xjtulyc Python Research Package Development
When to Use This Skill
Use this skill when you need to:
- Set up a modern Python package with
pyproject.toml(PEP 517/518) - Structure a research software repository with src layout
- Write unit and integration tests with pytest
- Generate API documentation with Sphinx and autodoc
- Publish a package to PyPI or a conda channel
- Set up continuous integration (GitHub Actions) for research software
- Create reproducible research packages with versioning
Trigger keywords: Python package, pyproject.toml, setuptools, build, wheel, pytest, coverage, Sphinx, autodoc, Read the Docs, PyPI, conda-forge, research software engineering, FAIR software, src layout, namespace package, entry points, semantic versioning.
Background & Key Concepts
Modern Python Packaging (PEP 517/518)
pyproject.toml is the single source of truth:
[build-system]: specifies build backend (setuptools, flit, poetry)[project]: package metadata (name, version, dependencies)[project.optional-dependencies]: optional extras[project.scripts]: CLI entry points
Semantic Versioning
MAJOR.MINOR.PATCH (e.g., 1.2.3):
- MAJOR: breaking API changes
- MINOR: backward-compatible new features
- PATCH: backward-compatible bug fixes
Use __version__ = "1.0.0" in src/mypackage/__init__.py.
Src Layout
Placing code in src/mypackage/ prevents accidental imports from the project root and enforces installation before use:
mypackage/
├── src/
│ └── mypackage/
│ ├── __init__.py
│ └── core.py
├── tests/
│ └── test_core.py
├── docs/
│ └── conf.py
└── pyproject.toml
Test Coverage
Coverage measures the fraction of code lines executed during tests. Target ≥ 80% for research packages. Use pytest-cov:
pytest --cov=src/mypackage --cov-report=html
Environment Setup
pip install build>=1.0 pytest>=7.4 pytest-cov>=4.1 sphinx>=7.2 \
twine>=4.0 setuptools>=68
# Verify tools
python -m pytest --version
python -m build --version
sphinx-build --version
Core Workflow
Step 1: Package Structure and pyproject.toml
# Create package structure
mkdir -p my_research_package/src/my_research_package
mkdir -p my_research_package/tests
mkdir -p my_research_package/docs
cd my_research_package
# File: src/my_research_package/__init__.py
"""My Research Package: Tools for quantitative analysis."""
__version__ = "0.1.0"
__author__ = "Research Team"
__email__ = "research@university.edu"
from .core import (
DataProcessor,
StatisticalAnalyzer,
compute_effect_size,
)
__all__ = ["DataProcessor", "StatisticalAnalyzer", "compute_effect_size"]
# File: src/my_research_package/core.py
"""Core analysis functions for my_research_package."""
import numpy as np
import pandas as pd
from typing import Optional, Union, List
class DataProcessor:
"""Preprocess and validate research datasets.
Args:
data: Input DataFrame or array-like
missing_strategy: How to handle missing values.
One of 'drop', 'mean', 'median', 'zero'.
"""
VALID_STRATEGIES = ("drop", "mean", "median", "zero")
def __init__(
self,
data: Union[pd.DataFrame, np.ndarray],
missing_strategy: str = "mean",
):
if missing_strategy not in self.VALID_STRATEGIES:
raise ValueError(
f"missing_strategy must be one of {self.VALID_STRATEGIES}, "
f"got '{missing_strategy}'"
)
self.missing_strategy = missing_strategy
self._data = self._validate_input(data)
def _validate_input(self, data):
"""Validate and convert input to DataFrame."""
if isinstance(data, pd.DataFrame):
return data.copy()
elif isinstance(data, np.ndarray):
if data.ndim == 1:
return pd.DataFrame(data, columns=["value"])
return pd.DataFrame(data)
else:
raise TypeError(f"Expected DataFrame or ndarray, got {type(data)}")
def process(self) -> pd.DataFrame:
"""Apply missing value strategy and return cleaned data.
Returns:
pd.DataFrame: Cleaned dataset.
"""
df = self._data.copy()
if self.missing_strategy == "drop":
return df.dropna()
elif self.missing_strategy == "mean":
return df.fillna(df.mean(numeric_only=True))
elif self.missing_strategy == "median":
return df.fillna(df.median(numeric_only=True))
elif self.missing_strategy == "zero":
return df.fillna(0)
@property
def shape(self):
"""Return (rows, columns) of the dataset."""
return self._data.shape
@property
def missing_count(self):
"""Return total number of missing values."""
return int(self._data.isna().sum().sum())
class StatisticalAnalyzer:
"""Compute descriptive and inferential statistics.
Args:
data: Input array-like of values
"""
def __init__(self, data: Union[np.ndarray, List[float], pd.Series]):
self.data = np.asarray(data, dtype=float)
def describe(self) -> dict:
"""Return descriptive statistics.
Returns:
dict: n, mean, std, min, q25, median, q75, max
"""
d = self.data[~np.isnan(self.data)]
if len(d) == 0:
return {}
return {
"n": len(d),
"mean": float(np.mean(d)),
"std": float(np.std(d, ddof=1)),
"min": float(np.min(d)),
"q25": float(np.percentile(d, 25)),
"median": float(np.median(d)),
"q75": float(np.percentile(d, 75)),
"max": float(np.max(d)),
}
def confidence_interval(self, alpha: float = 0.05) -> tuple:
"""Compute (1-alpha)% confidence interval for the mean.
Args:
alpha: Significance level (default 0.05 for 95% CI)
Returns:
(lower, upper) confidence interval tuple
"""
from scipy import stats
d = self.data[~np.isnan(self.data)]
n = len(d)
if n < 2:
raise ValueError("Need at least 2 observations for CI")
se = stats.sem(d)
h = se * stats.t.ppf(1 - alpha / 2, df=n - 1)
mean = np.mean(d)
return (float(mean - h), float(mean + h))
def compute_effect_size(
group1: Union[np.ndarray, List[float]],
group2: Union[np.ndarray, List[float]],
method: str = "cohens_d",
) -> float:
"""Compute effect size between two groups.
Args:
group1: Observations from first group
group2: Observations from second group
method: Effect size measure. One of 'cohens_d', 'glass_delta', 'hedges_g'
Returns:
Effect size value (positive = group1 > group2)
Raises:
ValueError: If method is not recognized
"""
g1 = np.asarray(group1, dtype=float)
g2 = np.asarray(group2, dtype=float)
mean_diff = np.mean(g1) - np.mean(g2)
n1, n2 = len(g1), len(g2)
if method == "cohens_d":
pooled_var = ((n1 - 1) * np.var(g1, ddof=1) +
(n2 - 1) * np.var(g2, ddof=1)) / (n1 + n2 - 2)
return float(mean_diff / np.sqrt(pooled_var))
elif method == "glass_delta":
return float(mean_diff / np.std(g2, ddof=1))
elif method == "hedges_g":
d = compute_effect_size(g1, g2, method="cohens_d")
j = 1 - 3 / (4 * (n1 + n2 - 2) - 1)
return float(d * j)
else:
raise ValueError(f"Unknown method '{method}'. "
f"Choose from: cohens_d, glass_delta, hedges_g")
# File: pyproject.toml
[build-system]
requires = ["setuptools>=68", "wheel"]
build-backend = "setuptools.backends.legacy:build"
[project]
name = "my-research-package"
version = "0.1.0"
description = "Tools for quantitative research analysis"
readme = "README.md"
license = {text = "MIT"}
authors = [
{name = "Research Team", email = "research@university.edu"},
]
keywords = ["research", "statistics", "data analysis"]
classifiers = [
"Development Status :: 3 - Alpha",
"Intended Audience :: Science/Research",
"License :: OSI Approved :: MIT License",
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3.9",
"Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11",
]
requires-python = ">=3.9"
dependencies = [
"numpy>=1.24",
"pandas>=2.0",
"scipy>=1.11",
]
[project.optional-dependencies]
viz = ["matplotlib>=3.7"]
dev = [
"pytest>=7.4",
"pytest-cov>=4.1",
"sphinx>=7.2",
"sphinx-rtd-theme>=1.3",
"twine>=4.0",
]
[project.urls]
Homepage = "https://github.com/example/my-research-package"
Documentation = "https://my-research-package.readthedocs.io"
Repository = "https://github.com/example/my-research-package"
[tool.setuptools.packages.find]
where = ["src"]
[tool.pytest.ini_options]
testpaths = ["tests"]
addopts = "--cov=src/my_research_package --cov-report=term-missing"
[tool.coverage.run]
source = ["src/my_research_package"]
[tool.coverage.report]
exclude_lines = [
"pragma: no cover",
"if __name__ == .__main__.:",
]
Step 2: Testing with Pytest
# File: tests/test_core.py
"""Tests for my_research_package core module."""
import pytest
import numpy as np
import pandas as pd
# Adjust import path if running tests standalone
import sys, os
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "src"))
from my_research_package.core import DataProcessor, StatisticalAnalyzer, compute_effect_size
class TestDataProcessor:
"""Tests for DataProcessor class."""
def setup_method(self):
"""Set up test fixtures."""
self.clean_data = pd.DataFrame({"a": [1.0, 2.0, 3.0], "b": [4.0, 5.0, 6.0]})
self.missing_data = pd.DataFrame({"a": [1.0, np.nan, 3.0], "b": [np.nan, 5.0, 6.0]})
def test_basic_initialization(self):
"""Test DataProcessor initializes with valid input."""
dp = DataProcessor(self.clean_data)
assert dp.shape == (3, 2)
assert dp.missing_count == 0
def test_missing_count(self):
"""Test missing count is correctly reported."""
dp = DataProcessor(self.missing_data)
assert dp.missing_count == 2
def test_mean_imputation(self):
"""Test mean imputation strategy."""
dp = DataProcessor(self.missing_data, missing_strategy="mean")
result = dp.process()
assert result.isna().sum().sum() == 0
assert result.loc[1, "a"] == pytest.approx(2.0) # mean of [1, 3]
def test_drop_strategy(self):
"""Test drop missing strategy removes rows with NaN."""
dp = DataProcessor(self.missing_data, missing_strategy="drop")
result = dp.process()
assert len(result) == 1 # only row index 2 has no NaN
def test_invalid_strategy_raises(self):
"""Test ValueError on invalid strategy."""
with pytest.raises(ValueError, match="missing_strategy must be one of"):
DataProcessor(self.clean_data, missing_strategy="invalid")
def test_numpy_array_input(self):
"""Test DataProcessor accepts numpy arrays."""
arr = np.array([1, 2, 3])
dp = DataProcessor(arr)
assert dp.shape == (3, 1)
def test_invalid_input_type_raises(self):
"""Test TypeError on invalid input type."""
with pytest.raises(TypeError):
DataProcessor([1, 2, 3]) # plain list not accepted
class TestStatisticalAnalyzer:
"""Tests for StatisticalAnalyzer class."""
def setup_method(self):
"""Set up test fixtures."""
np.random.seed(42)
self.data = np.random.normal(10, 2, 100)
def test_describe_output_keys(self):
"""Test describe() returns expected keys."""
sa = StatisticalAnalyzer(self.data)
result = sa.describe()
expected_keys = {"n", "mean", "std", "min", "q25", "median", "q75", "max"}
assert set(result.keys()) == expected_keys
def test_describe_mean_approx(self):
"""Test mean is approximately correct."""
sa = StatisticalAnalyzer(self.data)
result = sa.describe()
assert result["mean"] == pytest.approx(np.mean(self.data))
def test_confidence_interval_coverage(self):
"""Test CI width makes statistical sense."""
sa = StatisticalAnalyzer(self.data)
lo, hi = sa.confidence_interval(alpha=0.05)
assert lo < hi
assert lo < np.mean(self.data) < hi
def test_ci_requires_minimum_data(self):
"""Test CI raises ValueError for insufficient data."""
sa = StatisticalAnalyzer([5.0])
with pytest.raises(ValueError, match="at least 2"):
sa.confidence_interval()
def test_nan_ignored_in_describe(self):
"""Test NaN values are excluded from computations."""
data_with_nan = np.array([1.0, 2.0, np.nan, 4.0, 5.0])
sa = StatisticalAnalyzer(data_with_nan)
result = sa.describe()
assert result["n"] == 4
class TestComputeEffectSize:
"""Tests for compute_effect_size function."""
def setup_method(self):
"""Set up test groups with known effect size."""
np.random.seed(42)
self.g1 = np.random.normal(10, 2, 50)
self.g2 = np.random.normal(8, 2, 50) # true d ≈ 1.0
def test_cohens_d_positive(self):
"""Test Cohen's d is positive when group1 > group2."""
d = compute_effect_size(self.g1, self.g2, method="cohens_d")
assert d > 0
def test_cohens_d_symmetric(self):
"""Test that reversing groups negates the effect size."""
d1 = compute_effect_size(self.g1, self.g2, method="cohens_d")
d2 = compute_effect_size(self.g2, self.g1, method="cohens_d")
assert d1 == pytest.approx(-d2, abs=1e-10)
def test_hedges_g_smaller_than_d(self):
"""Test Hedges' g is smaller than Cohen's d in magnitude."""
d = abs(compute_effect_size(self.g1, self.g2, method="cohens_d"))
g = abs(compute_effect_size(self.g1, self.g2, method="hedges_g"))
assert g < d
def test_unknown_method_raises(self):
"""Test ValueError on unknown method."""
with pytest.raises(ValueError, match="Unknown method"):
compute_effect_size(self.g1, self.g2, method="unknown_es")
@pytest.mark.parametrize("method", ["cohens_d", "glass_delta", "hedges_g"])
def test_all_methods_return_float(self, method):
"""Test all methods return a float."""
result = compute_effect_size(self.g1, self.g2, method=method)
assert isinstance(result, float)
# Run tests with verbose output
if __name__ == "__main__":
pytest.main([__file__, "-v", "--tb=short"])
Step 3: Documentation Setup with Sphinx
# Initialize Sphinx docs
cd docs
sphinx-quickstart --quiet --project "My Research Package" \
--author "Research Team" --release "0.1.0" \
--language en --ext-autodoc --ext-napoleon \
--ext-viewcode --makefile --no-batchfile
# File: docs/conf.py (after sphinx-quickstart, add these settings)
import os
import sys
# Add src to path for autodoc
sys.path.insert(0, os.path.abspath("../src"))
project = "My Research Package"
copyright = "2026, Research Team"
author = "Research Team"
release = "0.1.0"
extensions = [
"sphinx.ext.autodoc", # auto-generate API docs from docstrings
"sphinx.ext.napoleon", # support NumPy/Google docstring styles
"sphinx.ext.viewcode", # add source code links
"sphinx.ext.intersphinx", # link to external docs (numpy, pandas)
"sphinx.ext.mathjax", # render LaTeX equations
]
# Napoleon settings for NumPy docstring style
napoleon_google_docstring = True
napoleon_numpy_docstring = True
napoleon_include_init_with_doc = False
napoleon_include_private_with_doc = False
# Autodoc settings
autodoc_default_options = {
"members": True,
"undoc-members": False,
"show-inheritance": True,
"special-members": "__init__",
}
# Intersphinx links
intersphinx_mapping = {
"python": ("https://docs.python.org/3", None),
"numpy": ("https://numpy.org/doc/stable", None),
"pandas": ("https://pandas.pydata.org/docs", None),
}
html_theme = "sphinx_rtd_theme"
.. File: docs/api.rst
API Reference
=============
Core Module
-----------
.. automodule:: my_research_package.core
:members:
:undoc-members:
:show-inheritance:
DataProcessor
~~~~~~~~~~~~~
.. autoclass:: my_research_package.core.DataProcessor
:members:
:special-members: __init__
StatisticalAnalyzer
~~~~~~~~~~~~~~~~~~~
.. autoclass:: my_research_package.core.StatisticalAnalyzer
:members:
:special-members: __init__
Functions
~~~~~~~~~
.. autofunction:: my_research_package.core.compute_effect_size
# Build HTML documentation
cd docs
make html
# Documentation generated in docs/_build/html/index.html
Advanced Usage
GitHub Actions CI/CD Pipeline
# File: .github/workflows/ci.yml
name: CI
on:
push:
branches: [main, develop]
pull_request:
branches: [main]
jobs:
test:
name: Test Python ${{ matrix.python-version }}
runs-on: ubuntu-latest
strategy:
matrix:
python-version: ["3.9", "3.10", "3.11"]
steps:
- uses: actions/checkout@v4
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Install package in development mode
run: |
python -m pip install --upgrade pip
pip install -e ".[dev]"
- name: Run tests with coverage
run: |
pytest --cov=src/my_research_package \
--cov-report=xml --cov-report=term-missing \
-v
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v3
with:
file: ./coverage.xml
fail_ci_if_error: false
publish:
name: Publish to PyPI
needs: test
runs-on: ubuntu-latest
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v4
with:
python-version: "3.11"
- name: Build distribution
run: |
pip install build
python -m build
- name: Publish to PyPI
uses: pypa/gh-action-pypi-publish@release/v1
with:
password: ${{ secrets.PYPI_API_TOKEN }}
Programmatic Version Bumping
# File: scripts/bump_version.py
"""Script to bump version in pyproject.toml and __init__.py."""
import re
import sys
import os
def bump_version(version_type="patch"):
"""Bump semantic version.
Args:
version_type: 'major', 'minor', or 'patch'
"""
# Read current version from __init__.py
init_path = os.path.join("src", "my_research_package", "__init__.py")
with open(init_path) as f:
content = f.read()
match = re.search(r'__version__ = "(\d+)\.(\d+)\.(\d+)"', content)
if not match:
raise ValueError("Could not find version string")
major, minor, patch = int(match.group(1)), int(match.group(2)), int(match.group(3))
if version_type == "major":
major += 1; minor = 0; patch = 0
elif version_type == "minor":
minor += 1; patch = 0
elif version_type == "patch":
patch += 1
else:
raise ValueError(f"Unknown version_type: {version_type}")
new_version = f"{major}.{minor}.{patch}"
new_content = content.replace(match.group(0), f'__version__ = "{new_version}"')
with open(init_path, "w") as f:
f.write(new_content)
# Update pyproject.toml
pyproject_path = "pyproject.toml"
with open(pyproject_path) as f:
proj_content = f.read()
old_ver_str = f'version = "{major-1 if version_type=="major" else major}.'
# Simple replacement (assumes version appears once)
old_v = match.group(0).replace("__version__ = ", "version = ")
proj_content = proj_content.replace(
f'version = "{match.group(1)}.{match.group(2)}.{match.group(3)}"',
f'version = "{new_version}"'
)
with open(pyproject_path, "w") as f:
f.write(proj_content)
print(f"Version bumped to {new_version}")
return new_version
if __name__ == "__main__":
vtype = sys.argv[1] if len(sys.argv) > 1 else "patch"
bump_version(vtype)
Troubleshooting
| Problem | Cause | Fix |
|---|---|---|
ModuleNotFoundError in tests |
src not in PYTHONPATH | Install in editable mode: pip install -e . |
| Sphinx autodoc finds no members | Wrong sys.path in conf.py | Verify sys.path.insert(0, "../src") |
| pytest import error | Conflicting package installations | Use python -m pytest instead of plain pytest |
| Build fails: missing wheel | build backend misconfigured | Install build: pip install build; run python -m build |
| PyPI upload fails | Token not set | Create API token at pypi.org; store as environment variable |
| Coverage < 80% | Untested branches | Add tests for edge cases; use --cov-branch flag |
External Resources
- Python Packaging User Guide
- pyproject.toml specification (PEP 518)
- pytest documentation
- Sphinx autodoc documentation
- Research Software Engineering best practices
- JOSS — Journal of Open Source Software
Examples
Example 1: CLI Entry Point for a Research Tool
# File: src/my_research_package/cli.py
"""Command-line interface for my_research_package."""
import argparse
import sys
import numpy as np
from .core import StatisticalAnalyzer, compute_effect_size
def main():
"""Entry point for the research analysis CLI."""
parser = argparse.ArgumentParser(
description="Compute statistics for research data",
formatter_class=argparse.RawDescriptionHelpFormatter,
)
subparsers = parser.add_subparsers(dest="command")
# Describe subcommand
desc_parser = subparsers.add_parser("describe", help="Compute descriptive statistics")
desc_parser.add_argument("--values", nargs="+", type=float, required=True,
help="Space-separated values")
# Effect size subcommand
es_parser = subparsers.add_parser("effect-size", help="Compute effect size")
es_parser.add_argument("--group1", nargs="+", type=float, required=True)
es_parser.add_argument("--group2", nargs="+", type=float, required=True)
es_parser.add_argument("--method", default="cohens_d",
choices=["cohens_d", "glass_delta", "hedges_g"])
args = parser.parse_args()
if args.command == "describe":
sa = StatisticalAnalyzer(args.values)
stats = sa.describe()
for k, v in stats.items():
print(f"{k}: {v:.4f}" if isinstance(v, float) else f"{k}: {v}")
elif args.command == "effect-size":
es = compute_effect_size(args.group1, args.group2, method=args.method)
print(f"{args.method} = {es:.4f}")
else:
parser.print_help()
sys.exit(1)
if __name__ == "__main__":
main()
# Add to pyproject.toml [project.scripts]
[project.scripts]
research-analyze = "my_research_package.cli:main"
Example 2: Automated Package Testing
# Run full test suite with coverage report
pytest tests/ -v --cov=src/my_research_package \
--cov-report=html --cov-report=term-missing \
--tb=short 2>&1 | head -50
# Build distribution packages
python -m build
ls dist/ # should show .tar.gz and .whl files
# Check distribution
twine check dist/*
# Install and verify
pip install dist/*.whl
python -c "import my_research_package; print(my_research_package.__version__)"