nature-academic-search

interface:
  display_name: "Nature Academic Search"
  short_description: "Search and verify scholarly literature across sources"
  default_prompt: "Use $nature-academic-search to search for relevant literature and verify every citation across independent sources."

{
  "mcpServers": {
    "academic-search": {
      "command": "uv",
      "args": [
        "run",
        "--no-project",
        "--directory",
        "<MCP_SERVER_DIR>",
        "--with",
        "mcp>=1.0.0,<2.0.0",
        "--with",
        "requests>=2.28.0,<3.0.0",
        "--with",
        "toml>=0.10.2,<2.0.0",
        "--with",
        "lxml>=4.9.0,<6.0.0",
        "--with",
        "pybliometrics>=4.4.1,<5.0.0",
        "python",
        "academic_search_server.py"
      ]
    }
  }
}

{
  "enabledMcpjsonServers": [
    "academic-search"
  ]
}

[academic-search]
trigger = ["查文献", "搜论文", "检索", "导入EndNote", "导出RIS", "检查参考文献", "验证DOI", "验证引用", "相关文献", "找类似论文", "构建检索式", "MeSH", "去重", "严格他引", "他引判定", "排除自引", "谁引用了我的文章", "引用我的文章的人有没有大牛", "文章引用表", "指定文章引用数", "严格他引数", "整理成表格", "院士引用", "校长引用", "院长引用", "杰青引用", "长江学者引用", "Fellow引用", "search papers", "find articles", "academic search", "literature search", "verify references", "check DOI", "verify citations", "strict other citation", "independent citation audit", "article citation table", "citation metrics table", "strict other-citation count", "influential citer", "citation context", "Fellow citation", "download citation", "export .nbib", "export .ris", "BibTeX", "DOI", "PMID"]
priority = 5
suppress_active_skills = ["lit-process"]

#!/usr/bin/env bash
# Academic Search Skill + MCP Server Installer for Claude Code
# Usage: bash install.sh [PUBMED_EMAIL]
set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
CLAUDE_DIR="${HOME}/.claude"
MCP_TARGET="${CLAUDE_DIR}/mcp_servers/academic-search"
SKILL_TARGET="${CLAUDE_DIR}/skills/academic-search"
MCP_JSON="${CLAUDE_DIR}/.mcp.json"

PUBMED_EMAIL="${1:[email protected]}"

echo "=== Academic Search Installer ==="
echo "Target: ${CLAUDE_DIR}"
echo "PubMed email: ${PUBMED_EMAIL}"
echo

required_paths=(
    "README.md"
    "SKILL.md"
    "manifest.yaml"
    "static"
    "references"
    "scripts"
    "config"
    "mcp-server"
)

missing_paths=()
for relpath in "${required_paths[@]}"; do
    if [ ! -e "${SCRIPT_DIR}/${relpath}" ]; then
        missing_paths+=("${relpath}")
    fi
done

if [ "${#missing_paths[@]}" -gt 0 ]; then
    echo "ERROR: incomplete nature-academic-search skill directory." >&2
    echo "Missing required path(s):" >&2
    for relpath in "${missing_paths[@]}"; do
        echo "  - ${SCRIPT_DIR}/${relpath}" >&2
    done
    echo "Install the full skills/nature-academic-search folder and retry." >&2
    exit 1
fi

# 1. Install Python dependencies
echo "[1/5] Installing Python dependencies..."
pip install --quiet -r "${SCRIPT_DIR}/mcp-server/requirements.txt" 2>/dev/null || {
    echo "  pip failed, trying pip3..."
    pip3 install --quiet -r "${SCRIPT_DIR}/mcp-server/requirements.txt" 2>/dev/null || {
        echo "  WARNING: Could not install Python deps. Install manually:"
        echo "    pip install -r mcp-server/requirements.txt"
    }
}

# 2. Copy MCP server
echo "[2/5] Copying MCP server..."
mkdir -p "${MCP_TARGET}"
cp -r "${SCRIPT_DIR}/mcp-server/"* "${MCP_TARGET}/"

# 3. Copy Skill
echo "[3/5] Copying Skill..."
mkdir -p "${SKILL_TARGET}"
cp "${SCRIPT_DIR}/README.md" "${SKILL_TARGET}/"
cp "${SCRIPT_DIR}/SKILL.md" "${SKILL_TARGET}/"
cp "${SCRIPT_DIR}/manifest.yaml" "${SKILL_TARGET}/"
cp -r "${SCRIPT_DIR}/static" "${SKILL_TARGET}/"
cp -r "${SCRIPT_DIR}/references" "${SKILL_TARGET}/"
cp -r "${SCRIPT_DIR}/scripts" "${SKILL_TARGET}/"
cp -r "${SCRIPT_DIR}/config" "${SKILL_TARGET}/"

# 4. Merge .mcp.json
echo "[4/5] Configuring .mcp.json..."
if [ -f "${MCP_JSON}" ]; then
    # Check if academic-search already exists
    if grep -q '"academic-search"' "${MCP_JSON}" 2>/dev/null; then
        echo "  academic-search already in .mcp.json, skipping merge."
    else
        # Inject into existing mcpServers object
        python3 -c "
import json, sys
with open('${MCP_JSON}', 'r') as f:
    cfg = json.load(f)
cfg.setdefault('mcpServers', {})['academic-search'] = {
    'command': 'python3',
    'args': ['${MCP_TARGET}/academic_search_server.py'],
    'env': {'PUBMED_EMAIL': '${PUBMED_EMAIL}'}
}
with open('${MCP_JSON}', 'w') as f:
    json.dump(cfg, f, indent=2)
    f.write('\n')
print('  Merged academic-search into existing .mcp.json')
"
    fi
else
    cat > "${MCP_JSON}" <<MCPJSON
{
  "mcpServers": {
    "academic-search": {
      "command": "python3",
      "args": ["${MCP_TARGET}/academic_search_server.py"],
      "env": {
        "PUBMED_EMAIL": "${PUBMED_EMAIL}"
      }
    }
  }
}
MCPJSON
    echo "  Created new .mcp.json"
fi

# 5. Enable in settings.json
echo "[5/5] Enabling in settings.json..."
SETTINGS_JSON="${CLAUDE_DIR}/settings.json"
if [ -f "${SETTINGS_JSON}" ]; then
    python3 -c "
import json
with open('${SETTINGS_JSON}', 'r') as f:
    cfg = json.load(f)
enabled = cfg.setdefault('enabledMcpjsonServers', [])
if 'academic-search' not in enabled:
    enabled.append('academic-search')
    with open('${SETTINGS_JSON}', 'w') as f:
        json.dump(cfg, f, indent=2)
        f.write('\n')
    print('  Added academic-search to enabledMcpjsonServers')
else:
    print('  academic-search already enabled')
"
else
    echo '  WARNING: settings.json not found. Manually add "academic-search" to enabledMcpjsonServers.'
fi

echo
echo "=== Done ==="
echo
echo "Installed:"
echo "  MCP server : ${MCP_TARGET}/"
echo "  Skill      : ${SKILL_TARGET}/"
echo
echo "Next steps:"
echo "  1. Restart Claude Code (or /clear)"
echo "  2. Set your PubMed email in config.toml or PUBMED_EMAIL env var"
echo "  3. (Optional) Add NCBI_API_KEY for higher rate limits"
echo "  4. Test: ask Claude 'search papers about CRISPR'"
echo
echo "Optional: copy triggers to your data/triggers.toml"
echo "  See: config/triggers-academic-search.toml"

name: nature-academic-search
version: 2.0.0
description: >
  Declarative manifest for the static/dynamic split. SKILL.md uses this to
  decide which fragments to load for a literature-search request. The main axis
  is the workflow: the user's need maps to one of six coordinated workflows,
  each already a self-contained file.

# Note on axis paths: the workflow files live in references/workflows/ and
# cross-reference the shared modules with ../ relative links. The workflow axis
# therefore points at them in place rather than moving them into
# static/fragments/, which would break those internal links. nature-academic-
# search does not use the prose-oriented nature-shared layer.

always_load:
  - static/core/tools.md
  - static/core/routing-and-ops.md

axes:
  workflow:
    detect: |
      Map the user's need to one workflow. Multiple may apply for a combined
      request (for example search then export); load each that applies.
        multi-source-search   — find literature across PubMed/CrossRef/arXiv and more
        citation-verification — verify or check citations extracted from a document
        mesh-strategy         — build a MeSH/PubMed search strategy
        citation-file-mgmt    — convert/manage .nbib/.ris/.bib citation files
        reference-mgmt        — BibTeX, related-article discovery, ID conversion
        strict-other-citation-impact-audit — audit strict independent citations,
                             influential citers, and citation context
    values:
      multi-source-search:   references/workflows/wf1-multi-source-search.md
      citation-verification: references/workflows/wf2-citation-verification.md
      mesh-strategy:         references/workflows/wf3-mesh-strategy.md
      citation-file-mgmt:    references/workflows/wf4-citation-file-mgmt.md
      reference-mgmt:        references/workflows/wf5-reference-mgmt.md
      strict-other-citation-impact-audit: references/workflows/wf6-strict-other-citation-impact-audit.md
    multi: true

references:
  on_demand:
    - condition: full source reliability tiers (T1/T2/T3) and fallback routing rules
      path: references/source-tiers.md
    - condition: deduplication across sources (used by WFs 1, 2, 5a)
      path: references/dedup-engine.md
    - condition: extracting citations from documents (WF 2)
      path: references/citation-parser.md
    - condition: query construction, source selection, and result ranking
      path: references/search-strategy.md
    - condition: strict other-citation, high-profile citer, and citation-context audit workflow
      path: references/workflows/wf6-strict-other-citation-impact-audit.md
    - condition: RIS/BibTeX format specifications and field mappings
      path: references/ris-bibtex-format.md
    - condition: no-MCP fallback discovery search (OpenAlex, stdlib) when the MCP server is unavailable
      path: scripts/academic_search.py
    - condition: multi-source .nbib/.ris/.bib downloading and conversion
      path: scripts/format-converter.py
    - condition: pre-flight check that API endpoints are reachable before batch operations
      path: scripts/preflight.py

"""Academic search MCP server.

Unified entry point exposing multi-source search and source-specific tools for
CrossRef, PubMed, arXiv, Scopus, and ScienceDirect.
"""

from __future__ import annotations

import asyncio
import json
import re
from typing import Any

from mcp.server import FastMCP

from sources import (
    ArxivSource,
    CrossRefSource,
    PubMedSource,
    ScienceDirectSource,
    ScopusSource,
)
from utils import AcademicSearchError, DataSourceError, setup_logging

mcp = FastMCP("academic-search")
logger = setup_logging()

# Singleton source instances (shared across tool calls)
_crossref = CrossRefSource()
_pubmed = PubMedSource()
_arxiv = ArxivSource()
_scopus = ScopusSource()
_sciencedirect = ScienceDirectSource()


# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------

def _detect_id_type(id: str) -> str:
    """Auto-detect identifier type.

    Returns one of: "doi", "pmid", "arxiv".
    Raises ValueError when detection fails.
    """
    id = id.strip()
    if id.startswith("10.") and "/" in id:
        return "doi"
    if re.match(r"^\d{7,8}$", id):
        return "pmid"
    if re.match(r"^\d{4}\.\d{4,5}(v\d+)?$", id):
        return "arxiv"
    raise ValueError(f"Cannot detect ID type for: {id}")


def _resolve_id_type(id: str, id_type: str) -> str:
    """Resolve the effective ID type.

    If id_type is "auto", delegate to _detect_id_type.
    Otherwise normalise the explicit type string.
    """
    if id_type == "auto":
        return _detect_id_type(id)
    normalised = id_type.lower().strip()
    if normalised in ("doi", "pmid", "arxiv"):
        return normalised
    raise ValueError(f"Unsupported id_type: {id_type}")


def _json_ok(data: Any) -> str:
    """Serialize a successful result to JSON string."""
    return json.dumps(data, ensure_ascii=False, indent=2)


def _json_error(message: str, source: str | None = None) -> str:
    """Serialize an error result to JSON string."""
    payload: dict[str, Any] = {"error": message}
    if source:
        payload["source"] = source
    return json.dumps(payload, ensure_ascii=False, indent=2)


# ---------------------------------------------------------------------------
# Async wrappers for synchronous sources
# ---------------------------------------------------------------------------

async def _search_crossref(query: str, rows: int, filter_type: str | None) -> dict:
    return await asyncio.to_thread(_crossref.search, query, rows, filter_type)


async def _search_pubmed(query: str, rows: int) -> dict:
    return await asyncio.to_thread(_pubmed.search, query, rows)


async def _search_arxiv(query: str, rows: int) -> dict:
    return await asyncio.to_thread(_arxiv.search, query, rows)


async def _search_scopus(query: str, rows: int) -> dict:
    return await asyncio.to_thread(_scopus.search, query, rows)


async def _search_sciencedirect(query: str, rows: int) -> dict:
    return await asyncio.to_thread(_sciencedirect.search, query, rows)


async def _search_all(
    query: str,
    sources: list[str],
    rows: int,
    filter_type: str | None,
) -> dict:
    """Dispatch concurrent searches and merge results."""
    tasks: list[asyncio.Task] = []
    source_order: list[str] = []

    if "crossref" in sources:
        tasks.append(asyncio.create_task(_search_crossref(query, rows, filter_type)))
        source_order.append("crossref")
    if "pubmed" in sources:
        tasks.append(asyncio.create_task(_search_pubmed(query, rows)))
        source_order.append("pubmed")
    if "arxiv" in sources:
        tasks.append(asyncio.create_task(_search_arxiv(query, rows)))
        source_order.append("arxiv")
    if "scopus" in sources:
        tasks.append(asyncio.create_task(_search_scopus(query, rows)))
        source_order.append("scopus")
    if "sciencedirect" in sources:
        tasks.append(asyncio.create_task(_search_sciencedirect(query, rows)))
        source_order.append("sciencedirect")

    if not tasks:
        return {"total": 0, "results": [], "errors": []}

    outcomes = await asyncio.gather(*tasks, return_exceptions=True)

    merged_results: list[dict] = []
    errors: list[dict] = []
    total = 0

    for src, outcome in zip(source_order, outcomes):
        if isinstance(outcome, BaseException):
            logger.error("Source %s failed: %s", src, outcome)
            errors.append({"source": src, "error": str(outcome)})
            continue
        total += outcome.get("total", 0)
        merged_results.extend(outcome.get("results", []))

    return {
        "total": total,
        "sources_queried": source_order,
        "result_count": len(merged_results),
        "results": merged_results,
        "errors": errors if errors else None,
    }


# ---------------------------------------------------------------------------
# Tools
# ---------------------------------------------------------------------------

@mcp.tool()
async def search_papers(
    query: str,
    sources: list[str] | None = None,
    rows: int = 5,
    type: str | None = None,
) -> str:
    """Search academic papers across multiple sources.

    Args:
        query: Search keywords or query string.
        sources: List of source names to query. Defaults to CrossRef, PubMed,
            and arXiv. Add "scopus" and/or "sciencedirect" explicitly to use
            Elsevier-backed providers; they require local pybliometrics config
            and may consume Elsevier API quota.
        rows: Number of results per source (max 50), not total result count.
        type: Optional CrossRef-only work type filter (e.g. "journal-article").

    Returns:
        JSON string with total count, merged results, and any per-source errors.
    """
    if not query or not query.strip():
        return _json_error("Empty search query")

    if sources is None:
        sources = ["crossref", "pubmed", "arxiv"]

    # Validate source names
    valid_sources = {"crossref", "pubmed", "arxiv", "scopus", "sciencedirect"}
    invalid = [s for s in sources if s not in valid_sources]
    if invalid:
        return _json_error(f"Invalid sources: {invalid}. Valid: {sorted(valid_sources)}")

    rows = max(1, min(rows, 50))

    logger.info("search_papers called", extra={
        "tool": "search_papers",
        "query": query,
        "sources": sources,
        "rows": rows,
    })

    try:
        result = await _search_all(query, sources, rows, type)
    except Exception as exc:
        logger.exception("search_papers failed")
        return _json_error(f"Search failed: {exc}")

    return _json_ok(result)


@mcp.tool()
def search_scopus(
    query: str,
    rows: int = 5,
    view: str | None = None,
    subscriber: bool = True,
) -> str:
    """Search Scopus documents using a Scopus advanced-search query.

    Args:
        query: Scopus advanced search query.
        rows: Number of normalized results to return (max 50).
        view: Optional Scopus view ("STANDARD" or "COMPLETE").
        subscriber: Whether to use subscriber cursor navigation.
    """
    try:
        return _json_ok(_scopus.search(query, rows, view=view, subscriber=subscriber))
    except DataSourceError as exc:
        logger.error("search_scopus failed: %s", exc)
        return _json_error(str(exc), source=exc.source)
    except Exception as exc:
        logger.exception("search_scopus failed unexpectedly")
        return _json_error(f"Unexpected error: {exc}")


@mcp.tool()
def get_scopus_abstract(
    identifier: str,
    id_type: str | None = None,
    view: str = "META_ABS",
) -> str:
    """Retrieve Scopus abstract metadata.

    Args:
        identifier: EID, Scopus ID, DOI, PMID, or PII.
        id_type: Optional pybliometrics ID type; auto-detected when omitted.
        view: Scopus abstract view, usually "META_ABS".
    """
    try:
        return _json_ok(_scopus.get_abstract(identifier, id_type=id_type, view=view))
    except DataSourceError as exc:
        logger.error("get_scopus_abstract failed: %s", exc)
        return _json_error(str(exc), source=exc.source)
    except Exception as exc:
        logger.exception("get_scopus_abstract failed unexpectedly")
        return _json_error(f"Unexpected error: {exc}")


@mcp.tool()
def get_scopus_citation_overview(
    identifiers: list[str],
    id_type: str = "scopus_id",
    date: str | None = None,
    citation: str | None = None,
) -> str:
    """Retrieve Scopus citation overview for one or more documents.

    Args:
        identifiers: Document identifiers.
        id_type: Identifier type, e.g. "scopus_id", "doi", or "eid".
        date: Optional year range such as "2020-2025".
        citation: Optional exclusion mode, e.g. "exclude-self".
    """
    try:
        result = _scopus.get_citation_overview(
            identifiers,
            id_type=id_type,
            date=date,
            citation=citation,
        )
        return _json_ok(result)
    except DataSourceError as exc:
        logger.error("get_scopus_citation_overview failed: %s", exc)
        return _json_error(str(exc), source=exc.source)
    except Exception as exc:
        logger.exception("get_scopus_citation_overview failed unexpectedly")
        return _json_error(f"Unexpected error: {exc}")


@mcp.tool()
def search_scopus_authors(query: str, rows: int = 5) -> str:
    """Search Scopus author profiles."""
    try:
        return _json_ok(_scopus.search_authors(query, rows))
    except DataSourceError as exc:
        logger.error("search_scopus_authors failed: %s", exc)
        return _json_error(str(exc), source=exc.source)
    except Exception as exc:
        logger.exception("search_scopus_authors failed unexpectedly")
        return _json_error(f"Unexpected error: {exc}")


@mcp.tool()
def get_scopus_author(author_id: str, view: str = "ENHANCED") -> str:
    """Retrieve a Scopus author profile by author ID."""
    try:
        return _json_ok(_scopus.get_author(author_id, view=view))
    except DataSourceError as exc:
        logger.error("get_scopus_author failed: %s", exc)
        return _json_error(str(exc), source=exc.source)
    except Exception as exc:
        logger.exception("get_scopus_author failed unexpectedly")
        return _json_error(f"Unexpected error: {exc}")


@mcp.tool()
def search_scopus_affiliations(query: str, rows: int = 5) -> str:
    """Search Scopus affiliations."""
    try:
        return _json_ok(_scopus.search_affiliations(query, rows))
    except DataSourceError as exc:
        logger.error("search_scopus_affiliations failed: %s", exc)
        return _json_error(str(exc), source=exc.source)
    except Exception as exc:
        logger.exception("search_scopus_affiliations failed unexpectedly")
        return _json_error(f"Unexpected error: {exc}")


@mcp.tool()
def get_scopus_affiliation(affiliation_id: str, view: str = "STANDARD") -> str:
    """Retrieve a Scopus affiliation by affiliation ID."""
    try:
        return _json_ok(_scopus.get_affiliation(affiliation_id, view=view))
    except DataSourceError as exc:
        logger.error("get_scopus_affiliation failed: %s", exc)
        return _json_error(str(exc), source=exc.source)
    except Exception as exc:
        logger.exception("get_scopus_affiliation failed unexpectedly")
        return _json_error(f"Unexpected error: {exc}")


@mcp.tool()
def search_scopus_serial_titles(
    title: str | None = None,
    issn: str | None = None,
    publisher: str | None = None,
    subject: str | None = None,
    subject_code: str | None = None,
    content: str | None = None,
    open_access: str | None = None,
    rows: int = 5,
    view: str = "ENHANCED",
) -> str:
    """Search Scopus serial titles.

    Args:
        title: Serial title query.
        issn: ISSN query.
        publisher: Publisher query.
        subject: Subject-area query.
        subject_code: Subject-area code query.
        content: Content type query.
        open_access: Open-access filter.
        rows: Number of results to return.
        view: Scopus serial title view.
    """
    query = {
        "title": title,
        "issn": issn,
        "pub": publisher,
        "subj": subject,
        "subjCode": subject_code,
        "content": content,
        "oa": open_access,
    }
    try:
        return _json_ok(_scopus.search_serial_titles(query, rows=rows, view=view))
    except DataSourceError as exc:
        logger.error("search_scopus_serial_titles failed: %s", exc)
        return _json_error(str(exc), source=exc.source)
    except Exception as exc:
        logger.exception("search_scopus_serial_titles failed unexpectedly")
        return _json_error(f"Unexpected error: {exc}")


@mcp.tool()
def get_scopus_serial_title(
    issn: str,
    view: str = "ENHANCED",
    years: str | None = None,
) -> str:
    """Retrieve a Scopus serial title by ISSN."""
    try:
        return _json_ok(_scopus.get_serial_title(issn, view=view, years=years))
    except DataSourceError as exc:
        logger.error("get_scopus_serial_title failed: %s", exc)
        return _json_error(str(exc), source=exc.source)
    except Exception as exc:
        logger.exception("get_scopus_serial_title failed unexpectedly")
        return _json_error(f"Unexpected error: {exc}")


@mcp.tool()
def get_scopus_plumx_metrics(identifier: str, id_type: str) -> str:
    """Retrieve PlumX metrics for a document identifier."""
    try:
        return _json_ok(_scopus.get_plumx_metrics(identifier, id_type))
    except DataSourceError as exc:
        logger.error("get_scopus_plumx_metrics failed: %s", exc)
        return _json_error(str(exc), source=exc.source)
    except Exception as exc:
        logger.exception("get_scopus_plumx_metrics failed unexpectedly")
        return _json_error(f"Unexpected error: {exc}")


@mcp.tool()
def search_sciencedirect(
    query: str,
    rows: int = 5,
    view: str | None = None,
) -> str:
    """Search ScienceDirect article metadata."""
    try:
        return _json_ok(_sciencedirect.search(query, rows=rows, view=view))
    except DataSourceError as exc:
        logger.error("search_sciencedirect failed: %s", exc)
        return _json_error(str(exc), source=exc.source)
    except Exception as exc:
        logger.exception("search_sciencedirect failed unexpectedly")
        return _json_error(f"Unexpected error: {exc}")


@mcp.tool()
def get_sciencedirect_article_metadata(
    query: str,
    rows: int = 5,
    view: str | None = None,
) -> str:
    """Retrieve ScienceDirect article metadata with a metadata API query."""
    try:
        result = _sciencedirect.get_article_metadata(query, rows=rows, view=view)
        return _json_ok(result)
    except DataSourceError as exc:
        logger.error("get_sciencedirect_article_metadata failed: %s", exc)
        return _json_error(str(exc), source=exc.source)
    except Exception as exc:
        logger.exception("get_sciencedirect_article_metadata failed unexpectedly")
        return _json_error(f"Unexpected error: {exc}")


@mcp.tool()
def get_paper_by_id(id: str, id_type: str = "auto") -> str:
    """Get paper details by identifier (DOI, PMID, or arXiv ID).

    Args:
        id: Paper identifier. Auto-detected if id_type is "auto":
            - Starts with "10." -> DOI (CrossRef)
            - 7-8 digit number -> PMID (PubMed)
            - YYMM.NNNNN format -> arXiv ID (arXiv)
        id_type: Force identifier type ("doi", "pmid", "arxiv", or "auto").

    Returns:
        JSON string with detailed paper metadata.
    """
    if not id or not id.strip():
        return _json_error("Empty identifier")

    try:
        resolved_type = _resolve_id_type(id, id_type)
    except ValueError as exc:
        return _json_error(str(exc))

    logger.info("get_paper_by_id called", extra={
        "tool": "get_paper_by_id",
        "id": id,
        "id_type": resolved_type,
    })

    try:
        if resolved_type == "doi":
            result = _crossref.get_by_doi(id.strip())
        elif resolved_type == "pmid":
            result = _pubmed.get_by_pmid(id.strip())
        elif resolved_type == "arxiv":
            result = _arxiv.get_by_id(id.strip())
        else:
            return _json_error(f"Unsupported ID type: {resolved_type}")
    except DataSourceError as exc:
        logger.error("get_paper_by_id failed: %s", exc)
        return _json_error(str(exc), source=exc.source)
    except Exception as exc:
        logger.exception("get_paper_by_id failed unexpectedly")
        return _json_error(f"Unexpected error: {exc}")

    return _json_ok(result)


@mcp.tool()
def get_citation(id: str, id_type: str = "auto", style: str = "apa") -> str:
    """Get formatted citation for a paper.

    Uses CrossRef content negotiation for DOI-based citations.
    For PMID/arXiv IDs, fetches metadata first then generates a basic citation.

    Args:
        id: Paper identifier (DOI, PMID, or arXiv ID).
        id_type: Force identifier type ("doi", "pmid", "arxiv", or "auto").
        style: Citation style. Supported: apa, nature, ieee, harvard,
               vancouver, chicago, mla.

    Returns:
        JSON string with the formatted citation.
    """
    if not id or not id.strip():
        return _json_error("Empty identifier")

    try:
        resolved_type = _resolve_id_type(id, id_type)
    except ValueError as exc:
        return _json_error(str(exc))

    logger.info("get_citation called", extra={
        "tool": "get_citation",
        "id": id,
        "id_type": resolved_type,
        "style": style,
    })

    try:
        if resolved_type == "doi":
            citation = _crossref.get_citation(id.strip(), style=style)
            return _json_ok({"id": id, "style": style, "citation": citation})

        # For non-DOI IDs, fetch metadata and build a basic citation
        if resolved_type == "pmid":
            paper = _pubmed.get_by_pmid(id.strip())
        elif resolved_type == "arxiv":
            paper = _arxiv.get_by_id(id.strip())
        else:
            return _json_error(f"Unsupported ID type: {resolved_type}")

        citation = _format_basic_citation(paper, style)
        return _json_ok({"id": id, "style": style, "citation": citation})

    except DataSourceError as exc:
        logger.error("get_citation failed: %s", exc)
        return _json_error(str(exc), source=exc.source)
    except Exception as exc:
        logger.exception("get_citation failed unexpectedly")
        return _json_error(f"Unexpected error: {exc}")


def _format_basic_citation(paper: dict, style: str) -> str:
    """Generate a basic citation string from unified paper metadata.

    This is a fallback for non-DOI papers where CrossRef content
    negotiation is not available.
    """
    authors = paper.get("authors", [])
    title = paper.get("title", "Untitled")
    year = paper.get("year", "n.d.")
    journal = paper.get("journal", "")
    doi = paper.get("doi", "")
    arxiv_id = paper.get("arxiv_id", "")
    pmid = paper.get("pmid", "")

    # Author formatting
    if len(authors) > 3:
        author_str = f"{authors[0]} et al."
    elif authors:
        author_str = ", ".join(authors)
    else:
        author_str = "Unknown"

    if style == "nature":
        parts = [f"{author_str}. {title}."]
        if journal:
            parts.append(f" *{journal}*.")
        if year:
            parts.append(f" ({year}).")
        if doi:
            parts.append(f" https://doi.org/{doi}")
        return "".join(parts)

    if style == "ieee":
        ref = f"{author_str}, \"{title}\""
        if journal:
            ref += f", *{journal}*"
        if year:
            ref += f", {year}"
        ref += "."
        if doi:
            ref += f" doi: {doi}."
        return ref

    # Default APA-like
    parts = [f"{author_str} ({year}). {title}."]
    if journal:
        parts.append(f" *{journal}*.")
    if doi:
        parts.append(f" https://doi.org/{doi}")
    elif arxiv_id:
        parts.append(f" arXiv:{arxiv_id}")
    elif pmid:
        parts.append(f" PMID:{pmid}")
    return "".join(parts)


@mcp.tool()
def lookup_mesh(term: str) -> str:
    """Lookup MeSH (Medical Subject Headings) terms.

    Queries the MeSH database via NCBI E-utilities to find matching
    descriptor names and unique IDs.

    Args:
        term: Search term to look up in the MeSH vocabulary.

    Returns:
        JSON string with matching MeSH descriptors.
    """
    if not term or not term.strip():
        return _json_error("Empty MeSH lookup term")

    logger.info("lookup_mesh called", extra={
        "tool": "lookup_mesh",
        "term": term,
    })

    try:
        result = _pubmed.lookup_mesh(term.strip())
    except DataSourceError as exc:
        logger.error("lookup_mesh failed: %s", exc)
        return _json_error(str(exc), source=exc.source)
    except Exception as exc:
        logger.exception("lookup_mesh failed unexpectedly")
        return _json_error(f"Unexpected error: {exc}")

    return _json_ok(result)


# ---------------------------------------------------------------------------
# Entry point
# ---------------------------------------------------------------------------

if __name__ == "__main__":
    mcp.run(transport="stdio")

[pubmed]
email = ""  # Set your PubMed email here or via PUBMED_EMAIL env var
api_key = ""
cache_dir = "~/.cache/academic-search"
cache_ttl = 86400

[crossref]
mailto = ""
timeout = 15

[arxiv]
timeout = 30

[general]
default_rows = 5
max_rows = 50

# 统一学术搜索 MCP 服务器

这是统一的学术搜索 MCP 服务器，整合 CrossRef、PubMed、arXiv、Scopus 和 ScienceDirect 数据源。

## 工具

| 工具 | 功能 |
|------|------|
| `search_papers` | 统一搜索，支持多数据源并发 |
| `get_paper_by_id` | 按 DOI、PMID 或 arXiv ID 获取详情 |
| `get_citation` | 格式化引用，支持 APA、Nature、IEEE 等风格 |
| `lookup_mesh` | MeSH 词表查询 |
| `search_scopus` | Scopus 高级检索 |
| `get_scopus_abstract` | Scopus 摘要与详情元数据 |
| `get_scopus_citation_overview` | Scopus 引用概览 |
| `search_scopus_authors` / `get_scopus_author` | 作者检索与详情 |
| `search_scopus_affiliations` / `get_scopus_affiliation` | 机构检索与详情 |
| `search_scopus_serial_titles` / `get_scopus_serial_title` | 期刊与连续出版物检索和详情 |
| `get_scopus_plumx_metrics` | PlumX 指标 |
| `search_sciencedirect` | ScienceDirect 检索 |
| `get_sciencedirect_article_metadata` | ScienceDirect 文章元数据 |

## 配置

环境变量：

- `PUBMED_EMAIL`：必填，NCBI 要求。
- `NCBI_API_KEY`：可选，用于提升速率限制。
- Elsevier / Scopus / ScienceDirect：复用 `pybliometrics` 配置文件，默认位置为 `~/.config/pybliometrics.cfg`。

`search_papers` 默认检索 CrossRef、PubMed 和 arXiv。Scopus / ScienceDirect 是可选 provider：只有在 `sources` 显式传入 `scopus` / `sciencedirect`，或调用专用 Scopus / ScienceDirect 工具时，才会访问 Elsevier API。

这样可以避免默认搜索无意消耗 Elsevier API 配额；若本机缺少 `pybliometrics` 配置，会在返回 JSON 的 `errors` 字段中给出对应数据源错误。

配置文件：`config.toml`

## 使用

插件会通过以下形式启动隔离运行环境：

```bash
uv run --no-project --directory <mcp-server> --with ... python academic_search_server.py
```

这些工具由 `academic-search` skill 调用。

mcp>=1.0.0,<2.0.0
requests>=2.28.0,<3.0.0
toml>=0.10.2,<2.0.0
lxml>=4.9.0,<6.0.0
pybliometrics>=4.4.1,<5.0.0

"""Data source modules for academic search."""

from .crossref import CrossRefSource
from .pubmed import PubMedSource
from .arxiv import ArxivSource
from .scopus import ScopusSource
from .sciencedirect import ScienceDirectSource

__all__ = [
    "CrossRefSource",
    "PubMedSource",
    "ArxivSource",
    "ScopusSource",
    "ScienceDirectSource",
]

"""arXiv data source via REST API (Atom XML feed)."""

import re
import time
import urllib.parse
import urllib.request
import xml.etree.ElementTree as ET
from datetime import datetime

from utils.config import get_config
from utils.errors import DataSourceError
from utils.logging import setup_logging

ARXIV_API_URL = "https://export.arxiv.org/api/query"

ARXIV_NS = {
    "atom": "http://www.w3.org/2005/Atom",
    "arxiv": "http://arxiv.org/schemas/atom",
}

_MIN_REQUEST_INTERVAL = 3.0  # seconds between requests
_SOURCE_NAME = "arxiv"

logger = setup_logging("INFO")


class ArxivSource:
    """arXiv search and retrieval via the Atom API."""

    def __init__(self):
        self._last_request_time: float = 0.0
        self._timeout: int | None = None

    # ------------------------------------------------------------------
    # Public interface
    # ------------------------------------------------------------------

    def search(
        self,
        query: str,
        rows: int = 5,
        categories: list[str] | None = None,
        date_from: str | None = None,
        date_to: str | None = None,
    ) -> dict:
        """Search arXiv.

        Parameters
        ----------
        query : str
            Free-text search query.
        rows : int
            Max number of results to return.
        categories : list[str] | None
            arXiv categories to restrict to (e.g. ["cs.AI", "cs.LG"]).
        date_from : str | None
            Start date in YYYY-MM-DD format.
        date_to : str | None
            End date in YYYY-MM-DD format.

        Returns
        -------
        dict
            {"results": [...], "total": int, "source": "arxiv"}
        """
        search_query = self._build_query(query, categories)
        date_filter = self._build_date_filter(date_from, date_to)

        params = {
            "search_query": search_query,
            "start": 0,
            "max_results": rows,
            "sortBy": "relevance",
            "sortOrder": "descending",
        }
        if date_filter:
            params["search_query"] = f"{search_query}+AND+{date_filter}"

        raw = self._request(params)
        results = self._parse_feed(raw)
        return {
            "results": results,
            "total": len(results),
            "source": _SOURCE_NAME,
        }

    def get_by_id(self, arxiv_id: str) -> dict:
        """Retrieve a single paper by arXiv ID.

        Parameters
        ----------
        arxiv_id : str
            arXiv identifier, e.g. "2401.12345" or "2401.12345v1".

        Returns
        -------
        dict
            Paper record in unified format.
        """
        clean_id = self._normalize_id(arxiv_id)
        params = {
            "id_list": clean_id,
            "max_results": 1,
        }
        raw = self._request(params)
        results = self._parse_feed(raw)
        if not results:
            raise DataSourceError(
                _SOURCE_NAME,
                f"Paper not found: {arxiv_id}",
            )
        return results[0]

    # ------------------------------------------------------------------
    # Query construction
    # ------------------------------------------------------------------

    @staticmethod
    def _build_query(user_query: str, categories: list[str] | None = None) -> str:
        """Build the search_query parameter.

        Combines user query with optional category restrictions.
        Returns a string with +AND+/+OR+ connectors (URL-ready).
        """
        parts: list[str] = []
        parts.append(f"({user_query})")
        if categories:
            cat_expr = " OR ".join(f"cat:{cat}" for cat in categories)
            parts.append(f"({cat_expr})")
        combined = " AND ".join(parts)
        # Replace connectors and spaces for URL embedding
        combined = (
            combined.replace(" AND ", "+AND+")
            .replace(" OR ", "+OR+")
            .replace(" ", "+")
        )
        return combined

    @staticmethod
    def _build_date_filter(
        date_from: str | None, date_to: str | None
    ) -> str:
        """Build submittedDate filter expression.

        The arXiv API requires the literal +TO+ syntax and 14-digit
        timestamps in the format YYYYMMDDHHMM.

        Returns empty string when no date bounds are given.
        """
        if not date_from and not date_to:
            return ""

        # Default boundaries
        start_ts = "000000000000"
        end_ts = "999912312359"

        if date_from:
            start_ts = _date_to_ts(date_from)
        if date_to:
            end_ts = _date_to_ts(date_to, end_of_day=True)

        return f"submittedDate:[{start_ts}+TO+{end_ts}]"

    # ------------------------------------------------------------------
    # HTTP
    # ------------------------------------------------------------------

    def _request(self, params: dict) -> str:
        """Execute an HTTP GET to the arXiv API with rate limiting."""
        self._enforce_rate_limit()
        url = f"{ARXIV_API_URL}?{urllib.parse.urlencode(params)}"
        timeout = self._get_timeout()
        logger.debug("arXiv request: %s", url)

        try:
            req = urllib.request.Request(url)
            req.add_header("User-Agent", "academic-search/1.0")
            with urllib.request.urlopen(req, timeout=timeout) as resp:
                return resp.read().decode("utf-8")
        except urllib.error.HTTPError as exc:
            if exc.code in (429, 503):
                raise DataSourceError(
                    _SOURCE_NAME,
                    f"Rate limited or unavailable (HTTP {exc.code})",
                    original_error=exc,
                ) from exc
            raise DataSourceError(
                _SOURCE_NAME,
                f"HTTP error {exc.code}: {exc.reason}",
                original_error=exc,
            ) from exc
        except urllib.error.URLError as exc:
            raise DataSourceError(
                _SOURCE_NAME,
                f"Network error: {exc.reason}",
                original_error=exc,
            ) from exc
        except TimeoutError as exc:
            raise DataSourceError(
                _SOURCE_NAME,
                f"Request timed out after {timeout}s",
                original_error=exc,
            ) from exc

    def _enforce_rate_limit(self) -> None:
        elapsed = time.monotonic() - self._last_request_time
        if elapsed < _MIN_REQUEST_INTERVAL:
            time.sleep(_MIN_REQUEST_INTERVAL - elapsed)
        self._last_request_time = time.monotonic()

    def _get_timeout(self) -> int:
        if self._timeout is None:
            self._timeout = get_config().arxiv_timeout
        return self._timeout

    # ------------------------------------------------------------------
    # XML parsing
    # ------------------------------------------------------------------

    def _parse_feed(self, xml_text: str) -> list[dict]:
        """Parse arXiv Atom XML into a list of unified result dicts."""
        try:
            root = ET.fromstring(xml_text)
        except ET.ParseError as exc:
            raise DataSourceError(
                _SOURCE_NAME,
                f"Malformed XML response: {exc}",
                original_error=exc,
            ) from exc

        entries: list[dict] = []
        for entry in root.findall("atom:entry", ARXIV_NS):
            parsed = self._parse_entry(entry)
            if parsed:
                entries.append(parsed)
        return entries

    def _parse_entry(self, entry: ET.Element) -> dict | None:
        """Extract a single paper record from an Atom <entry>."""
        arxiv_id_raw = _text(entry, "atom:id", ARXIV_NS)
        if not arxiv_id_raw:
            return None

        title = _text(entry, "atom:title", ARXIV_NS) or ""
        title = re.sub(r"\s+", " ", title).strip()

        summary = _text(entry, "atom:summary", ARXIV_NS) or ""
        summary = re.sub(r"\s+", " ", summary).strip()

        authors = [
            name
            for name in (
                _text(author, "atom:name", ARXIV_NS)
                for author in entry.findall("atom:author", ARXIV_NS)
            )
            if name
        ]

        primary_cat = entry.find("arxiv:primary_category", ARXIV_NS)
        categories = []
        if primary_cat is not None:
            term = primary_cat.get("term")
            if term:
                categories.append(term)
        # Also collect secondary categories from atom:category elements
        for cat in entry.findall("atom:category", ARXIV_NS):
            scheme = cat.get("scheme", "")
            term = cat.get("term", "")
            if term and "arxiv" in scheme.lower() and term not in categories:
                categories.append(term)

        published = _text(entry, "atom:published", ARXIV_NS) or ""
        year = _extract_year(published)

        pdf_url = ""
        for link in entry.findall("atom:link", ARXIV_NS):
            if link.get("title") == "pdf":
                pdf_url = link.get("href", "")
                break

        arxiv_id = self._normalize_id(arxiv_id_raw)

        return {
            "title": title,
            "authors": authors,
            "year": year,
            "arxiv_id": arxiv_id,
            "categories": categories,
            "abstract": summary,
            "pdf_url": pdf_url,
            "source": _SOURCE_NAME,
        }

    # ------------------------------------------------------------------
    # Helpers
    # ------------------------------------------------------------------

    @staticmethod
    def _normalize_id(raw: str) -> str:
        """Strip URL prefix and version suffix from an arXiv ID.

        http://arxiv.org/abs/2401.12345v1 -> 2401.12345
        """
        arxiv_id = raw.strip()
        # Remove URL prefix
        match = re.search(r"(\d{4}\.\d{4,5})(v\d+)?$", arxiv_id)
        if match:
            return match.group(1)
        # Fallback: strip known prefixes
        for prefix in ("https://arxiv.org/abs/", "http://arxiv.org/abs/"):
            if arxiv_id.startswith(prefix):
                arxiv_id = arxiv_id[len(prefix):]
                break
        # Strip version
        arxiv_id = re.sub(r"v\d+$", "", arxiv_id)
        return arxiv_id


# ------------------------------------------------------------------
# Module-level helpers
# ------------------------------------------------------------------


def _text(parent: ET.Element, xpath: str, ns: dict) -> str | None:
    """Return stripped text of a sub-element, or None."""
    el = parent.find(xpath, ns)
    if el is not None and el.text:
        return el.text.strip()
    return None


def _date_to_ts(date_str: str, end_of_day: bool = False) -> str:
    """Convert YYYY-MM-DD to YYYYMMDDHHMM (14-digit timestamp).

    Parameters
    ----------
    date_str : str
        Date in YYYY-MM-DD format.
    end_of_day : bool
        If True, use 2359 as HHMM; otherwise 0000.
    """
    dt = datetime.strptime(date_str, "%Y-%m-%d")
    suffix = "2359" if end_of_day else "0000"
    return dt.strftime("%Y%m%d") + suffix


def _extract_year(published: str) -> int | None:
    """Extract year from ISO datetime string (e.g. 2024-01-15T...)."""
    try:
        return int(published[:4])
    except (ValueError, IndexError):
        return None

"""CrossRef data source for academic search."""

from urllib.parse import quote

import requests

from utils.config import get_config
from utils.errors import DataSourceError

CROSSREF_API = "https://api.crossref.org"


class CrossRefSource:
    """CrossRef API wrapper with unified result format."""

    SOURCE_NAME = "crossref"

    def __init__(self):
        config = get_config()
        mailto = config.crossref_mailto or "[email protected]"
        self._headers = {
            "User-Agent": f"ClaudeCode-MCP-Crossref/1.0 (mailto:{mailto})",
        }
        self._timeout = config.crossref_timeout

    # ------------------------------------------------------------------
    # Public API
    # ------------------------------------------------------------------

    def search(
        self, query: str, rows: int = 5, filter_type: str | None = None
    ) -> dict:
        """Search CrossRef works.

        Args:
            query: Keywords, author name, title, DOI prefix, etc.
            rows: Number of results (max 50).
            filter_type: Optional work type filter, e.g. "journal-article".

        Returns:
            {"total": int, "results": [unified_result, ...]}
        """
        params: dict = {"query": query, "rows": min(rows, 50)}
        if filter_type:
            params["filter"] = f"type:{filter_type}"

        data = self._request("/works", params=params)
        items = data.get("items", [])
        total = data.get("total-results", 0)

        results = [self._normalize_search_item(item) for item in items]
        return {"total": total, "results": results}

    def get_by_doi(self, doi: str) -> dict:
        """Get detailed metadata for a single work by DOI.

        Args:
            doi: Digital Object Identifier (e.g. "10.1038/nature12373").

        Returns:
            Unified result dict with extra fields (abstract, volume, etc.).
        """
        data = self._request(f"/works/{quote(doi, safe='/')}")
        return self._normalize_detail_item(data)

    def get_citation(self, doi: str, style: str = "apa") -> str:
        """Return a formatted citation string via CrossRef content negotiation.

        Args:
            doi: Digital Object Identifier.
            style: Citation style (apa, nature, vancouver, ieee, etc.).

        Returns:
            Formatted citation string.
        """
        url = f"{CROSSREF_API}/works/{quote(doi, safe='/')}/transform"
        headers = {
            **self._headers,
            "Accept": f"text/x-bibliography; style={style}",
        }
        try:
            resp = requests.get(url, headers=headers, timeout=self._timeout)
        except requests.RequestException as exc:
            raise DataSourceError(
                self.SOURCE_NAME,
                f"Network error fetching citation for {doi}: {exc}",
                original_error=exc,
            ) from exc

        if resp.status_code == 404:
            return f"Citation not available for DOI: {doi}"
        if resp.status_code == 406:
            raise DataSourceError(
                self.SOURCE_NAME,
                f"Unsupported citation style: {style}",
            )

        try:
            resp.raise_for_status()
        except requests.HTTPError as exc:
            raise DataSourceError(
                self.SOURCE_NAME,
                f"HTTP {resp.status_code} fetching citation for {doi}",
                original_error=exc,
            ) from exc

        return resp.text.strip()

    # ------------------------------------------------------------------
    # Internal helpers
    # ------------------------------------------------------------------

    def _request(self, path: str, params: dict | None = None) -> dict:
        """Issue GET to CrossRef API and return the ``message`` payload."""
        url = f"{CROSSREF_API}{path}"
        try:
            resp = requests.get(
                url, params=params, headers=self._headers, timeout=self._timeout
            )
            resp.raise_for_status()
        except requests.HTTPError as exc:
            status = exc.response.status_code if exc.response is not None else "?"
            raise DataSourceError(
                self.SOURCE_NAME,
                f"HTTP {status} from {url}",
                original_error=exc,
            ) from exc
        except requests.RequestException as exc:
            raise DataSourceError(
                self.SOURCE_NAME,
                f"Network error calling {url}: {exc}",
                original_error=exc,
            ) from exc

        return resp.json().get("message", {})

    # ------------------------------------------------------------------
    # Normalization
    # ------------------------------------------------------------------

    @staticmethod
    def _extract_authors(author_list: list[dict], limit: int = 0) -> list[str]:
        """Convert CrossRef author entries to ``["Given Family", ...]`` list.

        Args:
            author_list: Raw ``author`` array from CrossRef.
            limit: Max authors to include; 0 = all.
        """
        subset = author_list[:limit] if limit else author_list
        names = [
            f"{a.get('given', '')} {a.get('family', '')}".strip()
            for a in subset
        ]
        if limit and len(author_list) > limit:
            names.append("et al.")
        return names

    @staticmethod
    def _extract_year(item: dict) -> int | None:
        """Best-effort publication year extraction."""
        for key in ("published-print", "published-online", "created"):
            parts = item.get(key, {}).get("date-parts", [[None]])
            year = parts[0][0] if parts and parts[0] else None
            if year is not None:
                return year
        return None

    def _normalize_search_item(self, item: dict) -> dict:
        """Map a CrossRef work item to the unified search result format."""
        return {
            "title": (item.get("title") or [""])[0],
            "authors": self._extract_authors(item.get("author", []), limit=5),
            "year": self._extract_year(item),
            "doi": item.get("DOI"),
            "journal": (item.get("container-title") or [""])[0],
            "source": self.SOURCE_NAME,
            "citation_count": item.get("is-referenced-by-count", 0),
        }

    def _normalize_detail_item(self, item: dict) -> dict:
        """Map a CrossRef work item to the unified detail result format."""
        base = self._normalize_search_item(item)
        base.update({
            "authors": self._extract_authors(item.get("author", [])),
            "abstract": item.get("abstract", ""),
            "volume": item.get("volume", ""),
            "issue": item.get("issue", ""),
            "pages": item.get("page", ""),
            "publisher": item.get("publisher", ""),
            "type": item.get("type"),
            "references_count": item.get("references-count", 0),
            "url": item.get("URL"),
        })
        return base

"""Shared helpers for pybliometrics-backed Elsevier sources."""

from __future__ import annotations

from pathlib import Path
from threading import Lock
from typing import Any

from pybliometrics import init as pybliometrics_init
from pybliometrics.utils.constants import CONFIG_FILE

from utils.errors import DataSourceError


_init_lock = Lock()
_initialised = False


def ensure_pybliometrics_config(source: str) -> None:
    """Initialise pybliometrics from its configured file.

    pybliometrics creates a config interactively when the file is absent.  MCP
    servers cannot prompt, so missing or invalid config is reported explicitly.
    """
    config_path = Path(CONFIG_FILE)
    if not config_path.exists():
        raise DataSourceError(
            source,
            f"pybliometrics config not found at {config_path}",
        )

    global _initialised
    with _init_lock:
        if _initialised:
            return
        try:
            pybliometrics_init(config_path=config_path)
        except (FileNotFoundError, ValueError) as exc:
            raise DataSourceError(
                source,
                f"pybliometrics config is invalid at {config_path}: {exc}",
                original_error=exc,
            ) from exc
        _initialised = True


def record_to_dict(value: Any) -> Any:
    """Recursively convert pybliometrics records into JSON-safe structures."""
    if value is None or isinstance(value, (str, int, float, bool)):
        return value
    if hasattr(value, "_asdict"):
        return {k: record_to_dict(v) for k, v in value._asdict().items()}
    if isinstance(value, dict):
        return {str(k): record_to_dict(v) for k, v in value.items()}
    if isinstance(value, (list, tuple, set)):
        return [record_to_dict(v) for v in value]
    return value


def safe_attr(obj: Any, name: str) -> Any:
    """Read an optional pybliometrics property."""
    try:
        return getattr(obj, name)
    except (AttributeError, KeyError, TypeError):
        return None


def split_semicolon(value: str | None) -> list[str]:
    """Split pybliometrics semicolon-joined fields."""
    if not value:
        return []
    return [item.strip() for item in value.split(";") if item.strip()]


def year_from_date(value: str | None) -> int | None:
    """Extract a publication year from an ISO-like date string."""
    if not value or len(value) < 4:
        return None
    try:
        return int(value[:4])
    except ValueError:
        return None

"""PubMed data source via NCBI E-utilities API."""

from __future__ import annotations

import time
import xml.etree.ElementTree as ET
from typing import Any

import requests

from utils.config import get_config
from utils.errors import DataSourceError
from utils.logging import setup_logging

logger = setup_logging()

BASE_URL = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/"
SOURCE_NAME = "pubmed"

# Rate limit: 3 req/s without key, 10 req/s with key
_REQ_INTERVAL_WITH_KEY = 0.11
_REQ_INTERVAL_WITHOUT_KEY = 0.35

_last_request_ts: float = 0.0


def _throttle(api_key: str) -> None:
    """Enforce NCBI rate limits."""
    global _last_request_ts
    interval = _REQ_INTERVAL_WITH_KEY if api_key else _REQ_INTERVAL_WITHOUT_KEY
    elapsed = time.monotonic() - _last_request_ts
    if elapsed < interval:
        time.sleep(interval - elapsed)
    _last_request_ts = time.monotonic()


def _get(endpoint: str, params: dict[str, Any], timeout: int = 30) -> requests.Response:
    """Send GET request to NCBI E-utilities with throttling and error handling."""
    cfg = get_config()
    api_key = cfg.pubmed_api_key

    _throttle(api_key)

    merged = dict(params)
    if cfg.pubmed_email:
        merged["email"] = cfg.pubmed_email
    if api_key:
        merged["api_key"] = api_key

    url = BASE_URL + endpoint
    try:
        resp = requests.get(url, params=merged, timeout=timeout)
        resp.raise_for_status()
    except requests.Timeout as exc:
        raise DataSourceError(SOURCE_NAME, f"Request timed out: {url}", exc) from exc
    except requests.HTTPError as exc:
        status = exc.response.status_code if exc.response is not None else "?"
        raise DataSourceError(
            SOURCE_NAME, f"HTTP {status} from {url}", exc
        ) from exc
    except requests.RequestException as exc:
        raise DataSourceError(SOURCE_NAME, f"Request failed: {url}", exc) from exc

    return resp


def _parse_article(article: ET.Element) -> dict[str, Any]:
    """Parse a single PubmedArticle XML element into the unified result dict."""
    citation = article.find("MedlineCitation")
    if citation is None:
        raise DataSourceError(SOURCE_NAME, "Missing MedlineCitation in article XML")

    pmid_el = citation.find("PMID")
    pmid = pmid_el.text.strip() if pmid_el is not None and pmid_el.text else ""

    art = citation.find("Article")
    if art is None:
        raise DataSourceError(SOURCE_NAME, f"Missing Article for PMID {pmid}")

    # Title
    title_el = art.find("ArticleTitle")
    title = title_el.text.strip() if title_el is not None and title_el.text else ""

    # Authors
    authors: list[str] = []
    author_list = art.find("AuthorList")
    if author_list is not None:
        for author in author_list.findall("Author"):
            last = author.find("LastName")
            fore = author.find("ForeName")
            if last is not None and last.text:
                name = last.text.strip()
                if fore is not None and fore.text:
                    name = f"{name} {fore.text.strip()}"
                authors.append(name)
            elif (collective := author.find("CollectiveName")) is not None and collective.text:
                authors.append(collective.text.strip())

    # Abstract
    abstract_parts: list[str] = []
    abstract_el = art.find("Abstract")
    if abstract_el is not None:
        for text_el in abstract_el.findall("AbstractText"):
            label = text_el.get("Label", "")
            content = "".join(text_el.itertext()).strip()
            if label and content:
                abstract_parts.append(f"{label}: {content}")
            elif content:
                abstract_parts.append(content)
    abstract = " ".join(abstract_parts)

    # Journal
    journal_el = art.find("Journal")
    journal = ""
    year = None
    if journal_el is not None:
        title_el = journal_el.find("Title")
        if title_el is not None and title_el.text:
            journal = title_el.text.strip()
        # Year from JournalIssue/PubDate
        issue = journal_el.find("JournalIssue")
        if issue is not None:
            pub_date = issue.find("PubDate")
            if pub_date is not None:
                year_el = pub_date.find("Year")
                if year_el is not None and year_el.text:
                    try:
                        year = int(year_el.text.strip())
                    except ValueError:
                        pass
                if year is None:
                    medline_date = pub_date.find("MedlineDate")
                    if medline_date is not None and medline_date.text:
                        # Extract first 4-digit year from string like "2024 Jan-Feb"
                        import re

                        m = re.search(r"\d{4}", medline_date.text)
                        if m:
                            year = int(m.group())

    # DOI
    doi = ""
    for eloi in art.findall("ELocationID"):
        if eloi.get("EIdType") == "doi" and eloi.text:
            doi = eloi.text.strip()
            break

    return {
        "title": title,
        "authors": authors,
        "year": year,
        "pmid": pmid,
        "doi": doi,
        "journal": journal,
        "abstract": abstract,
        "source": SOURCE_NAME,
    }


class PubMedSource:
    """PubMed data source providing search, fetch, and MeSH lookup."""

    def search(
        self,
        query: str,
        rows: int = 5,
        sort: str = "relevance",
    ) -> dict[str, Any]:
        """Search PubMed and return structured results.

        Args:
            query: PubMed search query string.
            rows: Number of results to return.
            sort: Sort order -- "relevance" (Best Match) or "date".

        Returns:
            Dict with keys: total, query, results (list of unified result dicts).
        """
        if not query or not query.strip():
            raise DataSourceError(SOURCE_NAME, "Empty search query")

        cfg = get_config()
        if not cfg.pubmed_email:
            raise DataSourceError(
                SOURCE_NAME,
                "PubMed email not configured. Set PUBMED_EMAIL env var or [pubmed].email in config.toml",
            )

        rows = min(rows, cfg.max_rows)
        sort_param = "relevance" if sort == "relevance" else "pub_date"

        # Step 1: esearch to get WebEnv + query_key
        search_params: dict[str, Any] = {
            "db": "pubmed",
            "term": query.strip(),
            "retmax": rows,
            "usehistory": "y",
            "retmode": "xml",
            "sort": sort_param,
        }
        resp = _get("esearch.fcgi", search_params)
        root = ET.fromstring(resp.content)

        count_el = root.find("Count")
        total = int(count_el.text) if count_el is not None and count_el.text else 0

        web_env_el = root.find("WebEnv")
        query_key_el = root.find("QueryKey")

        if web_env_el is None or query_key_el is None or not web_env_el.text or not query_key_el.text:
            # No results
            return {"total": 0, "query": query, "results": []}

        web_env = web_env_el.text.strip()
        query_key = query_key_el.text.strip()

        # Step 2: efetch to get article details
        fetch_params: dict[str, Any] = {
            "db": "pubmed",
            "query_key": query_key,
            "WebEnv": web_env,
            "retmax": rows,
            "retmode": "xml",
            "rettype": "abstract",
        }
        resp = _get("efetch.fcgi", fetch_params)
        fetch_root = ET.fromstring(resp.content)

        results: list[dict[str, Any]] = []
        for article in fetch_root.findall("PubmedArticle"):
            try:
                results.append(_parse_article(article))
            except DataSourceError as exc:
                logger.warning("Failed to parse article: %s", exc)
                continue

        return {"total": total, "query": query, "results": results}

    def get_by_pmid(self, pmid: str) -> dict[str, Any]:
        """Fetch a single article by PMID.

        Args:
            pmid: PubMed ID (numeric string).

        Returns:
            Unified result dict for the article.

        Raises:
            DataSourceError: If PMID is invalid or article not found.
        """
        if not pmid or not pmid.strip().isdigit():
            raise DataSourceError(SOURCE_NAME, f"Invalid PMID: {pmid}")

        cfg = get_config()
        if not cfg.pubmed_email:
            raise DataSourceError(
                SOURCE_NAME,
                "PubMed email not configured. Set PUBMED_EMAIL env var or [pubmed].email in config.toml",
            )

        fetch_params: dict[str, Any] = {
            "db": "pubmed",
            "id": pmid.strip(),
            "retmode": "xml",
            "rettype": "abstract",
        }
        resp = _get("efetch.fcgi", fetch_params)
        root = ET.fromstring(resp.content)

        article = root.find("PubmedArticle")
        if article is None:
            raise DataSourceError(SOURCE_NAME, f"PMID {pmid} not found")

        return _parse_article(article)

    def lookup_mesh(self, term: str) -> dict[str, Any]:
        """Look up a MeSH descriptor by term.

        Queries the MeSH database via E-utilities to find matching
        descriptor names and unique IDs.

        Args:
            term: Search term to look up in MeSH.

        Returns:
            Dict with keys: term, results (list of {name, mesh_id, ui}).
        """
        if not term or not term.strip():
            raise DataSourceError(SOURCE_NAME, "Empty MeSH lookup term")

        cfg = get_config()
        if not cfg.pubmed_email:
            raise DataSourceError(
                SOURCE_NAME,
                "PubMed email not configured. Set PUBMED_EMAIL env var or [pubmed].email in config.toml",
            )

        # Use esearch on MeSH database
        search_params: dict[str, Any] = {
            "db": "mesh",
            "term": term.strip(),
            "retmax": 10,
            "retmode": "xml",
        }
        resp = _get("esearch.fcgi", search_params)
        root = ET.fromstring(resp.content)

        id_list = root.find("IdList")
        if id_list is None or len(id_list) == 0:
            return {"term": term, "results": []}

        ids = [id_el.text.strip() for id_el in id_list.findall("Id") if id_el.text]

        if not ids:
            return {"term": term, "results": []}

        # efetch from mesh db to get descriptor details
        fetch_params: dict[str, Any] = {
            "db": "mesh",
            "id": ",".join(ids),
            "retmode": "xml",
        }
        resp = _get("efetch.fcgi", fetch_params)
        fetch_root = ET.fromstring(resp.content)

        results: list[dict[str, str]] = []
        for descriptor in fetch_root.findall(".//DescriptorRecord"):
            name_el = descriptor.find("DescriptorName/String")
            ui_el = descriptor.find("DescriptorUI")
            name = name_el.text.strip() if name_el is not None and name_el.text else ""
            ui = ui_el.text.strip() if ui_el is not None and ui_el.text else ""
            if name:
                results.append({"name": name, "mesh_id": ui, "ui": ui})

        return {"term": term, "results": results}

"""ScienceDirect data source via pybliometrics."""

from __future__ import annotations

from typing import Any

from pybliometrics.utils import URLS, get_content

from utils.errors import DataSourceError

from .elsevier_common import ensure_pybliometrics_config, year_from_date


SOURCE_NAME = "sciencedirect"


class ScienceDirectSource:
    """pybliometrics-backed ScienceDirect metadata operations."""

    SOURCE_NAME = SOURCE_NAME

    def search(
        self,
        query: str,
        rows: int = 5,
        view: str | None = None,
        refresh: bool | int = False,
    ) -> dict[str, Any]:
        """Search ScienceDirect article metadata and return the requested page."""
        if not query or not query.strip():
            raise DataSourceError(SOURCE_NAME, "Empty search query")

        rows = max(1, min(rows, 50))

        def run() -> dict[str, Any]:
            data = self._search_api(
                "ScienceDirectSearch",
                {
                    "query": query.strip(),
                    "count": rows,
                    "start": 0,
                    "view": view or "STANDARD",
                },
            )
            records = _search_entries(data)
            return {
                "total": _total_results(data),
                "query": query,
                "source": SOURCE_NAME,
                "results": [self._normalize_search_entry(r) for r in records[:rows]],
            }

        return self._run("ScienceDirect search", run)

    def get_article_metadata(
        self,
        query: str,
        rows: int = 5,
        view: str | None = None,
        refresh: bool | int = False,
    ) -> dict[str, Any]:
        """Retrieve ScienceDirect article metadata using a metadata API query."""
        if not query or not query.strip():
            raise DataSourceError(SOURCE_NAME, "Empty article metadata query")

        rows = max(1, min(rows, 50))

        def run() -> dict[str, Any]:
            data = self._search_api(
                "ArticleMetadata",
                {
                    "query": query.strip(),
                    "count": rows,
                    "start": 0,
                    "view": view or "STANDARD",
                },
            )
            records = _search_entries(data)
            return {
                "total": _total_results(data),
                "query": query,
                "source": SOURCE_NAME,
                "results": [self._normalize_metadata_entry(r) for r in records[:rows]],
            }

        return self._run("ScienceDirect article metadata retrieval", run)

    def _run(self, operation: str, callback):
        ensure_pybliometrics_config(SOURCE_NAME)
        try:
            return callback()
        except DataSourceError:
            raise
        except Exception as exc:
            raise DataSourceError(
                SOURCE_NAME,
                f"{operation} failed: {exc}",
                original_error=exc,
            ) from exc

    @staticmethod
    def _search_api(api: str, params: dict[str, Any]) -> dict[str, Any]:
        response = get_content(URLS[api], api, params=params)
        return response.json()

    @staticmethod
    def _normalize_search_entry(data: dict[str, Any]) -> dict[str, Any]:
        links = _links(data.get("link"))
        return {
            "title": data.get("dc:title"),
            "authors": _authors(data),
            "year": year_from_date(data.get("prism:coverDate")),
            "doi": _doi(data),
            "pii": data.get("pii"),
            "journal": data.get("prism:publicationName"),
            "volume": data.get("prism:volume"),
            "pages": _join_pages(data.get("prism:startingPage"), data.get("prism:endingPage")),
            "openaccess_status": data.get("openaccess"),
            "link": links.get("scidir"),
            "api_link": links.get("self") or data.get("prism:url"),
            "source": SOURCE_NAME,
        }

    @staticmethod
    def _normalize_metadata_entry(data: dict[str, Any]) -> dict[str, Any]:
        links = _links(data.get("link"))
        return {
            "title": data.get("dc:title"),
            "authors": _authors(data),
            "year": year_from_date(data.get("prism:coverDate")),
            "doi": _doi(data),
            "eid": data.get("eid"),
            "pii": data.get("pii"),
            "abstract": data.get("dc:description"),
            "journal": data.get("prism:publicationName"),
            "pages": _join_pages(data.get("prism:startingPage"), data.get("prism:endingPage")),
            "aggregation_type": data.get("prism:aggregationType"),
            "publication_type": data.get("prism:publicationType"),
            "author_keywords": data.get("authkeywords"),
            "openaccess_status": data.get("openaccess"),
            "link": links.get("scidir") or links.get("self"),
            "api_link": links.get("self") or data.get("prism:url"),
            "source": SOURCE_NAME,
        }


def _search_entries(data: dict[str, Any]) -> list[dict[str, Any]]:
    entries = data.get("search-results", {}).get("entry", [])
    return [
        entry
        for entry in _as_list(entries)
        if isinstance(entry, dict)
        and (entry.get("dc:title") or entry.get("prism:doi") or entry.get("pii"))
    ]


def _total_results(data: dict[str, Any]) -> int:
    total = data.get("search-results", {}).get("opensearch:totalResults", 0)
    return _int_or_none(total) or 0


def _as_list(value: Any) -> list[Any]:
    if value is None:
        return []
    if isinstance(value, list):
        return value
    return [value]


def _authors(data: dict[str, Any]) -> list[str]:
    authors = []
    author_data = data.get("authors", {}).get("author", [])
    for item in _as_list(author_data):
        name = item.get("$") if isinstance(item, dict) else item
        if name:
            authors.append(name)
    creator = data.get("dc:creator")
    if isinstance(creator, list):
        authors.extend(item.get("$") for item in creator if isinstance(item, dict))
    elif isinstance(creator, str) and not authors:
        authors.append(creator)
    return [a for a in authors if a]


def _links(value: Any) -> dict[str, str]:
    out = {}
    for item in _as_list(value):
        if not isinstance(item, dict):
            continue
        ref = item.get("@ref") or item.get("rel")
        href = item.get("@href") or item.get("href")
        if ref and href:
            out[ref] = href
    return out


def _doi(data: dict[str, Any]) -> str | None:
    if data.get("prism:doi"):
        return data["prism:doi"]
    identifier = data.get("dc:identifier")
    if isinstance(identifier, str) and identifier.startswith("doi:"):
        return identifier[4:]
    return None


def _join_pages(start: str | None, end: str | None) -> str:
    if start and end:
        return f"{start}-{end}"
    return start or end or ""


def _int_or_none(value: Any) -> int | None:
    try:
        return int(value)
    except (TypeError, ValueError):
        return None

"""Scopus data source via pybliometrics."""

from __future__ import annotations

from typing import Any

from pybliometrics.scopus import (
    AbstractRetrieval,
    AffiliationRetrieval,
    AuthorRetrieval,
    CitationOverview,
    PlumXMetrics,
    SerialTitleISSN,
)
from pybliometrics.utils import URLS, get_content

from utils.errors import DataSourceError

from .elsevier_common import (
    ensure_pybliometrics_config,
    record_to_dict,
    safe_attr,
    year_from_date,
)


SOURCE_NAME = "scopus"


class ScopusSource:
    """pybliometrics-backed Scopus operations."""

    SOURCE_NAME = SOURCE_NAME

    def search(
        self,
        query: str,
        rows: int = 5,
        view: str | None = None,
        refresh: bool | int = False,
        subscriber: bool = True,
    ) -> dict[str, Any]:
        """Search Scopus documents and return only the requested first page."""
        if not query or not query.strip():
            raise DataSourceError(SOURCE_NAME, "Empty search query")

        rows = max(1, min(rows, 50))

        def run() -> dict[str, Any]:
            effective_view = view or ("COMPLETE" if subscriber else "STANDARD")
            data = self._search_api(
                "ScopusSearch",
                {"query": query.strip(), "count": rows, "start": 0, "view": effective_view},
            )
            records = _search_entries(data)
            return {
                "total": _total_results(data),
                "query": query,
                "source": SOURCE_NAME,
                "results": [self._normalize_search_entry(r) for r in records[:rows]],
            }

        return self._run("Scopus search", run)

    def get_abstract(
        self,
        identifier: str,
        id_type: str | None = None,
        view: str = "META_ABS",
        refresh: bool | int = False,
    ) -> dict[str, Any]:
        """Retrieve Scopus abstract metadata by EID, Scopus ID, DOI, PMID, or PII."""
        if not identifier or not identifier.strip():
            raise DataSourceError(SOURCE_NAME, "Empty identifier")

        def run() -> dict[str, Any]:
            paper = AbstractRetrieval(
                identifier.strip(),
                id_type=id_type,
                view=view,
                refresh=refresh,
            )
            return self._normalize_abstract(paper)

        return self._run("Scopus abstract retrieval", run)

    def get_citation_overview(
        self,
        identifiers: list[str],
        id_type: str = "scopus_id",
        date: str | None = None,
        citation: str | None = None,
        refresh: bool | int = False,
    ) -> dict[str, Any]:
        """Retrieve Scopus citation overview for one or more documents."""
        clean = [i.strip() for i in identifiers if i and i.strip()]
        if not clean:
            raise DataSourceError(SOURCE_NAME, "Empty identifier list")

        def run() -> dict[str, Any]:
            overview = CitationOverview(
                clean,
                date=date,
                id_type=id_type,
                citation=citation,
                refresh=refresh,
            )
            return self._normalize_citation_overview(overview)

        return self._run("Scopus citation overview", run)

    def search_authors(
        self,
        query: str,
        rows: int = 5,
        refresh: bool | int = False,
    ) -> dict[str, Any]:
        """Search Scopus author profiles and return only the requested first page."""
        if not query or not query.strip():
            raise DataSourceError(SOURCE_NAME, "Empty author search query")
        rows = max(1, min(rows, 50))

        def run() -> dict[str, Any]:
            data = self._search_api(
                "AuthorSearch",
                {"query": query.strip(), "count": rows, "start": 0},
            )
            records = _search_entries(data)
            return {
                "total": _total_results(data),
                "query": query,
                "source": SOURCE_NAME,
                "results": [self._normalize_author_entry(r) for r in records[:rows]],
            }

        return self._run("Scopus author search", run)

    def get_author(
        self,
        author_id: str,
        view: str = "ENHANCED",
        refresh: bool | int = False,
    ) -> dict[str, Any]:
        """Retrieve a Scopus author profile."""
        if not author_id or not author_id.strip():
            raise DataSourceError(SOURCE_NAME, "Empty author ID")

        def run() -> dict[str, Any]:
            author = AuthorRetrieval(author_id.strip(), view=view, refresh=refresh)
            return {
                "author_id": safe_attr(author, "identifier"),
                "eid": safe_attr(author, "eid"),
                "orcid": safe_attr(author, "orcid"),
                "surname": safe_attr(author, "surname"),
                "given_name": safe_attr(author, "given_name"),
                "indexed_name": safe_attr(author, "indexed_name"),
                "document_count": safe_attr(author, "document_count"),
                "citation_count": safe_attr(author, "citation_count"),
                "cited_by_count": safe_attr(author, "cited_by_count"),
                "h_index": safe_attr(author, "h_index"),
                "coauthor_count": safe_attr(author, "coauthor_count"),
                "publication_range": record_to_dict(safe_attr(author, "publication_range")),
                "affiliation_current": record_to_dict(safe_attr(author, "affiliation_current")),
                "affiliation_history": record_to_dict(safe_attr(author, "affiliation_history")),
                "subject_areas": record_to_dict(safe_attr(author, "subject_areas")),
                "scopus_author_link": safe_attr(author, "scopus_author_link"),
                "url": safe_attr(author, "url"),
                "source": SOURCE_NAME,
            }

        return self._run("Scopus author retrieval", run)

    def search_affiliations(
        self,
        query: str,
        rows: int = 5,
        refresh: bool | int = False,
    ) -> dict[str, Any]:
        """Search Scopus affiliations and return only the requested first page."""
        if not query or not query.strip():
            raise DataSourceError(SOURCE_NAME, "Empty affiliation search query")
        rows = max(1, min(rows, 50))

        def run() -> dict[str, Any]:
            data = self._search_api(
                "AffiliationSearch",
                {"query": query.strip(), "count": rows, "start": 0},
            )
            records = _search_entries(data)
            return {
                "total": _total_results(data),
                "query": query,
                "source": SOURCE_NAME,
                "results": [self._normalize_affiliation_entry(r) for r in records[:rows]],
            }

        return self._run("Scopus affiliation search", run)

    def get_affiliation(
        self,
        affiliation_id: str,
        view: str = "STANDARD",
        refresh: bool | int = False,
    ) -> dict[str, Any]:
        """Retrieve a Scopus affiliation profile."""
        if not affiliation_id or not affiliation_id.strip():
            raise DataSourceError(SOURCE_NAME, "Empty affiliation ID")

        def run() -> dict[str, Any]:
            affiliation = AffiliationRetrieval(
                affiliation_id.strip(),
                view=view,
                refresh=refresh,
            )
            return {
                "affiliation_id": safe_attr(affiliation, "identifier"),
                "eid": safe_attr(affiliation, "eid"),
                "name": safe_attr(affiliation, "affiliation_name"),
                "sort_name": safe_attr(affiliation, "sort_name"),
                "documents": safe_attr(affiliation, "document_count"),
                "authors": safe_attr(affiliation, "author_count"),
                "address": safe_attr(affiliation, "address"),
                "city": safe_attr(affiliation, "city"),
                "state": safe_attr(affiliation, "state"),
                "country": safe_attr(affiliation, "country"),
                "postal_code": safe_attr(affiliation, "postal_code"),
                "org_domain": safe_attr(affiliation, "org_domain"),
                "org_url": safe_attr(affiliation, "org_URL"),
                "variants": record_to_dict(safe_attr(affiliation, "name_variants")),
                "scopus_affiliation_link": safe_attr(affiliation, "scopus_affiliation_link"),
                "url": safe_attr(affiliation, "url"),
                "source": SOURCE_NAME,
            }

        return self._run("Scopus affiliation retrieval", run)

    def search_serial_titles(
        self,
        query: dict[str, str],
        rows: int = 5,
        view: str = "ENHANCED",
        refresh: bool | int = False,
    ) -> dict[str, Any]:
        """Search Scopus serial titles and return only the requested first page."""
        clean = {k: v for k, v in query.items() if v}
        if not clean:
            raise DataSourceError(SOURCE_NAME, "Empty serial title query")
        rows = max(1, min(rows, 50))

        def run() -> dict[str, Any]:
            data = self._search_api(
                "SerialTitleSearch",
                {**clean, "count": rows, "start": 0, "view": view},
            )
            records = data.get("serial-metadata-response", {}).get("entry", [])
            records = _as_list(records)
            return {
                "total": len(records),
                "query": clean,
                "source": SOURCE_NAME,
                "results": [record_to_dict(r) for r in records[:rows]],
            }

        return self._run("Scopus serial title search", run)

    def get_serial_title(
        self,
        issn: str,
        view: str = "ENHANCED",
        years: str | None = None,
        refresh: bool | int = False,
    ) -> dict[str, Any]:
        """Retrieve a Scopus serial title by ISSN."""
        if not issn or not issn.strip():
            raise DataSourceError(SOURCE_NAME, "Empty ISSN")

        def run() -> dict[str, Any]:
            title = SerialTitleISSN(
                issn.strip(),
                view=view,
                years=years,
                refresh=refresh,
            )
            return {
                "title": safe_attr(title, "title"),
                "source_id": safe_attr(title, "source_id"),
                "issn": safe_attr(title, "issn"),
                "eissn": safe_attr(title, "eissn"),
                "publisher": safe_attr(title, "publisher"),
                "aggregation_type": safe_attr(title, "aggregation_type"),
                "openaccess": safe_attr(title, "openaccess"),
                "subject_areas": record_to_dict(safe_attr(title, "subject_area")),
                "citescore_year_info": record_to_dict(
                    safe_attr(title, "citescoreyearinfolist")
                ),
                "scopus_source_link": safe_attr(title, "scopus_source_link"),
                "source": SOURCE_NAME,
            }

        return self._run("Scopus serial title retrieval", run)

    def get_plumx_metrics(
        self,
        identifier: str,
        id_type: str,
        refresh: bool | int = False,
    ) -> dict[str, Any]:
        """Retrieve PlumX metrics for a document."""
        if not identifier or not identifier.strip():
            raise DataSourceError(SOURCE_NAME, "Empty PlumX identifier")
        if not id_type or not id_type.strip():
            raise DataSourceError(SOURCE_NAME, "Empty PlumX id_type")

        def run() -> dict[str, Any]:
            metrics = PlumXMetrics(
                identifier.strip(),
                id_type.strip(),
                refresh=refresh,
            )
            return {
                "identifier": identifier,
                "id_type": id_type,
                "category_totals": record_to_dict(safe_attr(metrics, "category_totals")),
                "capture": record_to_dict(safe_attr(metrics, "capture")),
                "citation": record_to_dict(safe_attr(metrics, "citation")),
                "mention": record_to_dict(safe_attr(metrics, "mention")),
                "social_media": record_to_dict(safe_attr(metrics, "social_media")),
                "usage": record_to_dict(safe_attr(metrics, "usage")),
                "source": SOURCE_NAME,
            }

        return self._run("Scopus PlumX metrics", run)

    def _run(self, operation: str, callback):
        ensure_pybliometrics_config(SOURCE_NAME)
        try:
            return callback()
        except DataSourceError:
            raise
        except Exception as exc:
            raise DataSourceError(
                SOURCE_NAME,
                f"{operation} failed: {exc}",
                original_error=exc,
            ) from exc

    @staticmethod
    def _search_api(api: str, params: dict[str, Any]) -> dict[str, Any]:
        response = get_content(URLS[api], api, params=params)
        return response.json()

    @staticmethod
    def _normalize_search_entry(data: dict[str, Any]) -> dict[str, Any]:
        return {
            "title": data.get("dc:title"),
            "authors": _extract_authors(data),
            "year": year_from_date(data.get("prism:coverDate")),
            "doi": data.get("prism:doi"),
            "eid": data.get("eid"),
            "pii": data.get("pii"),
            "pmid": data.get("pubmed-id"),
            "journal": data.get("prism:publicationName"),
            "volume": data.get("prism:volume"),
            "issue": data.get("prism:issueIdentifier"),
            "pages": data.get("prism:pageRange"),
            "subtype": data.get("subtype"),
            "subtype_description": data.get("subtypeDescription"),
            "citation_count": _int_or_none(data.get("citedby-count")),
            "openaccess": data.get("openaccess"),
            "source": SOURCE_NAME,
        }

    @staticmethod
    def _normalize_author_entry(data: dict[str, Any]) -> dict[str, Any]:
        preferred = data.get("preferred-name", {})
        affiliation = data.get("affiliation-current", {})
        areas = _as_list(data.get("subject-area"))
        return {
            "eid": data.get("eid"),
            "orcid": data.get("orcid"),
            "surname": preferred.get("surname"),
            "initials": preferred.get("initials"),
            "givenname": preferred.get("given-name"),
            "affiliation": affiliation.get("affiliation-name"),
            "documents": _int_or_none(data.get("document-count")),
            "affiliation_id": affiliation.get("affiliation-id"),
            "city": affiliation.get("affiliation-city"),
            "country": affiliation.get("affiliation-country"),
            "areas": [
                {
                    "abbreviation": area.get("@abbrev"),
                    "frequency": _int_or_none(area.get("@frequency")),
                    "name": area.get("$"),
                }
                for area in areas
            ],
            "source": SOURCE_NAME,
        }

    @staticmethod
    def _normalize_affiliation_entry(data: dict[str, Any]) -> dict[str, Any]:
        variants = [
            item.get("$")
            for item in _as_list(data.get("name-variant"))
            if item.get("$") and item.get("$") != data.get("affiliation-name")
        ]
        return {
            "eid": data.get("eid"),
            "name": data.get("affiliation-name"),
            "variant": ";".join(variants),
            "documents": _int_or_none(data.get("document-count")),
            "city": data.get("city"),
            "country": data.get("country"),
            "source": SOURCE_NAME,
        }

    @staticmethod
    def _normalize_abstract(paper: Any) -> dict[str, Any]:
        return {
            "title": safe_attr(paper, "title"),
            "authors": record_to_dict(safe_attr(paper, "authors")),
            "year": year_from_date(safe_attr(paper, "coverDate")),
            "doi": safe_attr(paper, "doi"),
            "eid": safe_attr(paper, "eid"),
            "scopus_id": safe_attr(paper, "identifier"),
            "pii": safe_attr(paper, "pii"),
            "abstract": safe_attr(paper, "abstract") or safe_attr(paper, "description"),
            "journal": safe_attr(paper, "publicationName"),
            "volume": safe_attr(paper, "volume"),
            "issue": safe_attr(paper, "issueIdentifier"),
            "pages": safe_attr(paper, "pageRange"),
            "publisher": safe_attr(paper, "publisher"),
            "citation_count": safe_attr(paper, "citedby_count"),
            "reference_count": safe_attr(paper, "refcount"),
            "affiliations": record_to_dict(safe_attr(paper, "affiliation")),
            "author_keywords": safe_attr(paper, "authkeywords"),
            "subject_areas": record_to_dict(safe_attr(paper, "subject_areas")),
            "url": safe_attr(paper, "url"),
            "source": SOURCE_NAME,
        }

    @staticmethod
    def _normalize_citation_overview(overview: Any) -> dict[str, Any]:
        titles = safe_attr(overview, "title") or []
        documents = []
        parallel_fields = {
            "scopus_id": safe_attr(overview, "scopus_id"),
            "doi": safe_attr(overview, "doi"),
            "title": titles,
            "publication_name": safe_attr(overview, "publicationName"),
            "row_total": safe_attr(overview, "rowTotal"),
            "range_count": safe_attr(overview, "rangeCount"),
            "citation_type": safe_attr(overview, "citationType_long"),
            "yearly_citations": safe_attr(overview, "cc"),
        }
        for idx in range(len(titles)):
            doc = {}
            for key, values in parallel_fields.items():
                if values is not None and idx < len(values):
                    doc[key] = record_to_dict(values[idx])
            documents.append(doc)

        return {
            "grand_total": safe_attr(overview, "grandTotal"),
            "h_index": safe_attr(overview, "h_index"),
            "column_total": safe_attr(overview, "columnTotal"),
            "previous_column_total": safe_attr(overview, "prevColumnTotal"),
            "range_column_total": safe_attr(overview, "rangeColumnTotal"),
            "later_column_total": safe_attr(overview, "laterColumnTotal"),
            "documents": documents,
            "source": SOURCE_NAME,
        }


def _search_entries(data: dict[str, Any]) -> list[dict[str, Any]]:
    entries = data.get("search-results", {}).get("entry", [])
    return _as_list(entries)


def _total_results(data: dict[str, Any]) -> int:
    total = data.get("search-results", {}).get("opensearch:totalResults", 0)
    return _int_or_none(total) or 0


def _as_list(value: Any) -> list[Any]:
    if value is None:
        return []
    if isinstance(value, list):
        return value
    return [value]


def _extract_authors(data: dict[str, Any]) -> list[str]:
    authors = []
    for item in _as_list(data.get("author")):
        name = item.get("authname") or item.get("ce:indexed-name")
        if name:
            authors.append(name)
    creator = data.get("dc:creator")
    if not authors and creator:
        authors.append(creator)
    return authors


def _int_or_none(value: Any) -> int | None:
    try:
        return int(value)
    except (TypeError, ValueError):
        return None

"""Tests for academic search server."""

"""Live API tests for Scopus and ScienceDirect.

These tests intentionally call Elsevier APIs through the local pybliometrics
configuration. They are not mocked.
"""

from __future__ import annotations

import asyncio
import json
import os

import pytest
from sources import ScienceDirectSource, ScopusSource


pytestmark = pytest.mark.skipif(
    os.getenv("NATURE_ACADEMIC_SEARCH_LIVE_ELSEVIER") != "1",
    reason="set NATURE_ACADEMIC_SEARCH_LIVE_ELSEVIER=1 to run live Elsevier API tests",
)

ELSEVIER_DOI = "10.1016/j.istruc.2024.107944"
ELSEVIER_TITLE_QUERY = (
    "Seismic performance continuous rigid-frame bridges flood-induced scour"
)


def _call_tool_json(tool_name: str, arguments: dict) -> dict:
    from academic_search_server import mcp

    content, _metadata = asyncio.run(mcp.call_tool(tool_name, arguments))
    return json.loads(content[0].text)


def test_scopus_live_search_exact_doi():
    result = ScopusSource().search(f"DOI({ELSEVIER_DOI})", rows=1)

    assert result["source"] == "scopus"
    assert result["total"] >= 1
    assert len(result["results"]) == 1
    assert result["results"][0]["doi"] == ELSEVIER_DOI


def test_sciencedirect_live_search_title_query():
    result = ScienceDirectSource().search(ELSEVIER_TITLE_QUERY, rows=1)

    assert result["source"] == "sciencedirect"
    assert result["total"] >= 1
    assert len(result["results"]) == 1
    assert result["results"][0]["doi"] == ELSEVIER_DOI


def test_sciencedirect_live_article_metadata_doi_query():
    result = ScienceDirectSource().get_article_metadata(
        f"doi({ELSEVIER_DOI})",
        rows=1,
    )

    assert result["source"] == "sciencedirect"
    assert result["total"] >= 1
    assert len(result["results"]) == 1
    assert result["results"][0]["doi"] == ELSEVIER_DOI


def test_default_search_papers_uses_free_sources_only():
    payload = _call_tool_json(
        "search_papers",
        {"query": ELSEVIER_TITLE_QUERY, "rows": 1},
    )

    assert payload["sources_queried"] == ["crossref", "pubmed", "arxiv"]
    assert "scopus" not in payload["sources_queried"]
    assert "sciencedirect" not in payload["sources_queried"]


def test_explicit_search_papers_includes_elsevier_sources():
    payload = _call_tool_json(
        "search_papers",
        {
            "query": ELSEVIER_TITLE_QUERY,
            "sources": ["scopus", "sciencedirect"],
            "rows": 1,
        },
    )

    assert payload["sources_queried"] == ["scopus", "sciencedirect"]
    assert any(item["source"] == "scopus" for item in payload["results"])
    assert any(item["source"] == "sciencedirect" for item in payload["results"])

"""MCP dispatch tests for academic_search_server tools."""

from __future__ import annotations

import asyncio
import json


def _call_tool_json(tool_name: str, arguments: dict) -> dict:
    from academic_search_server import mcp

    content, _metadata = asyncio.run(mcp.call_tool(tool_name, arguments))
    return json.loads(content[0].text)


def test_search_papers_mcp_dispatch_uses_default_sources(monkeypatch):
    import academic_search_server

    captured = {}

    async def fake_search_all(query, sources, rows, filter_type):
        captured.update({
            "query": query,
            "sources": list(sources),
            "rows": rows,
            "filter_type": filter_type,
        })
        return {
            "total": 0,
            "sources_queried": list(sources),
            "result_count": 0,
            "results": [],
            "errors": None,
        }

    monkeypatch.setattr(academic_search_server, "_search_all", fake_search_all)

    payload = _call_tool_json("search_papers", {"query": "graphene", "rows": 100})

    assert payload["sources_queried"] == ["crossref", "pubmed", "arxiv"]
    assert "error" not in payload
    assert captured == {
        "query": "graphene",
        "sources": ["crossref", "pubmed", "arxiv"],
        "rows": 50,
        "filter_type": None,
    }


def test_search_papers_mcp_dispatch_accepts_elsevier_sources(monkeypatch):
    import academic_search_server

    captured = {}

    async def fake_search_all(query, sources, rows, filter_type):
        captured.update({
            "query": query,
            "sources": list(sources),
            "rows": rows,
            "filter_type": filter_type,
        })
        return {
            "total": 2,
            "sources_queried": list(sources),
            "result_count": 2,
            "results": [
                {"source": "scopus", "title": "Scopus result"},
                {"source": "sciencedirect", "title": "ScienceDirect result"},
            ],
            "errors": None,
        }

    monkeypatch.setattr(academic_search_server, "_search_all", fake_search_all)

    payload = _call_tool_json(
        "search_papers",
        {
            "query": "bridge scour",
            "sources": ["scopus", "sciencedirect"],
            "rows": 1,
            "type": "journal-article",
        },
    )

    assert payload["sources_queried"] == ["scopus", "sciencedirect"]
    assert {item["source"] for item in payload["results"]} == {
        "scopus",
        "sciencedirect",
    }
    assert captured == {
        "query": "bridge scour",
        "sources": ["scopus", "sciencedirect"],
        "rows": 1,
        "filter_type": "journal-article",
    }

"""Unit tests for academic search source modules and ID detection.

All external HTTP calls are mocked -- no network access required.
"""

from __future__ import annotations

import json
import re
import xml.etree.ElementTree as ET
from unittest.mock import MagicMock, patch

import pytest

# ---------------------------------------------------------------------------
# Fixtures / helpers
# ---------------------------------------------------------------------------


def _make_crossref_config():
    """Return a mock Config object for CrossRef."""
    cfg = MagicMock()
    cfg.crossref_mailto = "[email protected]"
    cfg.crossref_timeout = 10
    return cfg


def _make_pubmed_config():
    """Return a mock Config object for PubMed."""
    cfg = MagicMock()
    cfg.pubmed_email = "[email protected]"
    cfg.pubmed_api_key = ""
    cfg.max_rows = 50
    return cfg


def _make_arxiv_config():
    """Return a mock Config object for arXiv."""
    cfg = MagicMock()
    cfg.arxiv_timeout = 10
    return cfg


# ===================================================================
# 1. CrossRef tests
# ===================================================================


class TestCrossRefSearch:
    """Test CrossRef search returns the unified result format."""

    @patch("sources.crossref.get_config")
    @patch("sources.crossref.requests.get")
    def test_search_returns_unified_format(self, mock_get, mock_config):
        mock_config.return_value = _make_crossref_config()

        mock_resp = MagicMock()
        mock_resp.status_code = 200
        mock_resp.raise_for_status = MagicMock()
        mock_resp.json.return_value = {
            "message": {
                "total-results": 42,
                "items": [
                    {
                        "title": ["Deep Learning for NLP"],
                        "author": [
                            {"given": "Alice", "family": "Smith"},
                            {"given": "Bob", "family": "Jones"},
                        ],
                        "published-print": {"date-parts": [[2023, 6, 15]]},
                        "DOI": "10.1234/example.2023",
                        "container-title": ["Journal of AI Research"],
                        "is-referenced-by-count": 17,
                    }
                ],
            }
        }
        mock_get.return_value = mock_resp

        from sources.crossref import CrossRefSource

        source = CrossRefSource()
        result = source.search("deep learning", rows=5)

        assert "total" in result
        assert "results" in result
        assert result["total"] == 42
        assert len(result["results"]) == 1

        item = result["results"][0]
        assert item["title"] == "Deep Learning for NLP"
        assert item["authors"] == ["Alice Smith", "Bob Jones"]
        assert item["year"] == 2023
        assert item["doi"] == "10.1234/example.2023"
        assert item["journal"] == "Journal of AI Research"
        assert item["source"] == "crossref"
        assert item["citation_count"] == 17

    @patch("sources.crossref.get_config")
    @patch("sources.crossref.requests.get")
    def test_search_with_type_filter(self, mock_get, mock_config):
        mock_config.return_value = _make_crossref_config()

        mock_resp = MagicMock()
        mock_resp.status_code = 200
        mock_resp.raise_for_status = MagicMock()
        mock_resp.json.return_value = {"message": {"total-results": 0, "items": []}}
        mock_get.return_value = mock_resp

        from sources.crossref import CrossRefSource

        source = CrossRefSource()
        result = source.search("test", rows=5, filter_type="journal-article")

        called_params = mock_get.call_args[1]["params"]
        assert called_params["filter"] == "type:journal-article"
        assert result["total"] == 0
        assert result["results"] == []

    @patch("sources.crossref.get_config")
    @patch("sources.crossref.requests.get")
    def test_search_empty_items(self, mock_get, mock_config):
        mock_config.return_value = _make_crossref_config()

        mock_resp = MagicMock()
        mock_resp.status_code = 200
        mock_resp.raise_for_status = MagicMock()
        mock_resp.json.return_value = {
            "message": {"total-results": 0, "items": []}
        }
        mock_get.return_value = mock_resp

        from sources.crossref import CrossRefSource

        source = CrossRefSource()
        result = source.search("nonexistent query xyz")

        assert result["total"] == 0
        assert result["results"] == []

    @patch("sources.crossref.get_config")
    @patch("sources.crossref.requests.get")
    def test_get_by_doi(self, mock_get, mock_config):
        mock_config.return_value = _make_crossref_config()

        mock_resp = MagicMock()
        mock_resp.status_code = 200
        mock_resp.raise_for_status = MagicMock()
        mock_resp.json.return_value = {
            "message": {
                "title": ["A Great Paper"],
                "author": [{"given": "Jane", "family": "Doe"}],
                "published-online": {"date-parts": [[2024]]},
                "DOI": "10.1038/nature12373",
                "container-title": ["Nature"],
                "abstract": "<p>We discovered something.</p>",
                "volume": "615",
                "issue": "7951",
                "page": "100-105",
                "publisher": "Springer Nature",
                "type": "journal-article",
                "references-count": 42,
                "URL": "https://doi.org/10.1038/nature12373",
            }
        }
        mock_get.return_value = mock_resp

        from sources.crossref import CrossRefSource

        source = CrossRefSource()
        detail = source.get_by_doi("10.1038/nature12373")

        assert detail["title"] == "A Great Paper"
        assert detail["doi"] == "10.1038/nature12373"
        assert detail["abstract"] == "<p>We discovered something.</p>"
        assert detail["volume"] == "615"
        assert detail["type"] == "journal-article"
        assert detail["source"] == "crossref"

    @patch("sources.crossref.get_config")
    @patch("sources.crossref.requests.get")
    def test_search_http_error(self, mock_get, mock_config):
        import requests as real_requests

        mock_config.return_value = _make_crossref_config()

        mock_resp = MagicMock()
        mock_resp.status_code = 503
        mock_resp.raise_for_status.side_effect = real_requests.HTTPError(
            response=mock_resp
        )
        mock_get.return_value = mock_resp

        from sources.crossref import CrossRefSource
        from utils.errors import DataSourceError

        source = CrossRefSource()
        with pytest.raises(DataSourceError, match="crossref"):
            source.search("test")

    @patch("sources.crossref.get_config")
    @patch("sources.crossref.requests.get")
    def test_get_citation(self, mock_get, mock_config):
        mock_config.return_value = _make_crossref_config()

        mock_resp = MagicMock()
        mock_resp.status_code = 200
        mock_resp.raise_for_status = MagicMock()
        mock_resp.text = "Smith, A. (2023). Deep Learning. Nature."
        mock_get.return_value = mock_resp

        from sources.crossref import CrossRefSource

        source = CrossRefSource()
        citation = source.get_citation("10.1038/nature12373", style="nature")

        assert "Smith" in citation
        assert "2023" in citation


# ===================================================================
# 2. PubMed tests
# ===================================================================


class TestPubMedSearch:
    """Test PubMed esearch + efetch flow with WebEnv/query_key handling."""

    @patch("sources.pubmed.get_config")
    @patch("sources.pubmed._get")
    def test_search_esearch_efetch_flow(self, mock_get, mock_config):
        mock_config.return_value = _make_pubmed_config()

        # esearch response
        esearch_xml = """<?xml version="1.0"?>
        <eSearchResult>
            <Count>1</Count>
            <RetMax>1</RetMax>
            <WebEnv>ABC123_webenv</WebEnv>
            <QueryKey>1</QueryKey>
            <IdList><Id>99999999</Id></IdList>
        </eSearchResult>"""

        # efetch response
        efetch_xml = """<?xml version="1.0"?>
        <PubmedArticleSet>
            <PubmedArticle>
                <MedlineCitation>
                    <PMID>99999999</PMID>
                    <Article>
                        <ArticleTitle>Genomic Analysis of Cancer</ArticleTitle>
                        <AuthorList>
                            <Author>
                                <LastName>Wang</LastName>
                                <ForeName>Li</ForeName>
                            </Author>
                            <Author>
                                <LastName>Zhang</LastName>
                                <ForeName>Wei</ForeName>
                            </Author>
                        </AuthorList>
                        <Abstract>
                            <AbstractText>We performed whole-genome sequencing.</AbstractText>
                        </Abstract>
                        <Journal>
                            <Title>Nature Medicine</Title>
                            <JournalIssue>
                                <PubDate><Year>2024</Year></PubDate>
                            </JournalIssue>
                        </Journal>
                        <ELocationID EIdType="doi">10.1038/s41591-024-00001</ELocationID>
                    </Article>
                </MedlineCitation>
            </PubmedArticle>
        </PubmedArticleSet>"""

        esearch_resp = MagicMock()
        esearch_resp.content = esearch_xml.encode("utf-8")

        efetch_resp = MagicMock()
        efetch_resp.content = efetch_xml.encode("utf-8")

        mock_get.side_effect = [esearch_resp, efetch_resp]

        from sources.pubmed import PubMedSource

        source = PubMedSource()
        result = source.search("cancer genomics", rows=5)

        assert result["total"] == 1
        assert result["query"] == "cancer genomics"
        assert len(result["results"]) == 1

        item = result["results"][0]
        assert item["title"] == "Genomic Analysis of Cancer"
        assert item["authors"] == ["Wang Li", "Zhang Wei"]
        assert item["year"] == 2024
        assert item["pmid"] == "99999999"
        assert item["doi"] == "10.1038/s41591-024-00001"
        assert item["journal"] == "Nature Medicine"
        assert item["source"] == "pubmed"
        assert "whole-genome" in item["abstract"]

        # Verify WebEnv and query_key were passed to efetch
        # _get(endpoint, params) -- params is positional arg index 1
        efetch_call_args = mock_get.call_args_list[1]
        efetch_params = efetch_call_args[0][1]
        assert efetch_params["WebEnv"] == "ABC123_webenv"
        assert efetch_params["query_key"] == "1"

    @patch("sources.pubmed.get_config")
    @patch("sources.pubmed._get")
    def test_search_no_results(self, mock_get, mock_config):
        mock_config.return_value = _make_pubmed_config()

        esearch_xml = """<?xml version="1.0"?>
        <eSearchResult>
            <Count>0</Count>
            <RetMax>0</RetMax>
            <IdList></IdList>
        </eSearchResult>"""

        mock_resp = MagicMock()
        mock_resp.content = esearch_xml.encode("utf-8")
        mock_get.return_value = mock_resp

        from sources.pubmed import PubMedSource

        source = PubMedSource()
        result = source.search("xyznonexistent12345", rows=5)

        assert result["total"] == 0
        assert result["results"] == []

    @patch("sources.pubmed.get_config")
    @patch("sources.pubmed._get")
    def test_get_by_pmid(self, mock_get, mock_config):
        mock_config.return_value = _make_pubmed_config()

        efetch_xml = """<?xml version="1.0"?>
        <PubmedArticleSet>
            <PubmedArticle>
                <MedlineCitation>
                    <PMID>12345678</PMID>
                    <Article>
                        <ArticleTitle>CRISPR Gene Editing Review</ArticleTitle>
                        <AuthorList>
                            <Author>
                                <LastName>Chen</LastName>
                                <ForeName>Xiaoming</ForeName>
                            </Author>
                        </AuthorList>
                        <Abstract>
                            <AbstractText>A comprehensive review of CRISPR.</AbstractText>
                        </Abstract>
                        <Journal>
                            <Title>Cell</Title>
                            <JournalIssue>
                                <PubDate><Year>2023</Year></PubDate>
                            </JournalIssue>
                        </Journal>
                    </Article>
                </MedlineCitation>
            </PubmedArticle>
        </PubmedArticleSet>"""

        mock_resp = MagicMock()
        mock_resp.content = efetch_xml.encode("utf-8")
        mock_get.return_value = mock_resp

        from sources.pubmed import PubMedSource

        source = PubMedSource()
        result = source.get_by_pmid("12345678")

        assert result["title"] == "CRISPR Gene Editing Review"
        assert result["pmid"] == "12345678"
        assert result["journal"] == "Cell"
        assert result["source"] == "pubmed"

    @patch("sources.pubmed.get_config")
    def test_search_empty_query_raises(self, mock_config):
        mock_config.return_value = _make_pubmed_config()

        from sources.pubmed import PubMedSource
        from utils.errors import DataSourceError

        source = PubMedSource()
        with pytest.raises(DataSourceError, match="Empty"):
            source.search("")

    @patch("sources.pubmed.get_config")
    def test_search_no_email_raises(self, mock_config):
        cfg = MagicMock()
        cfg.pubmed_email = ""
        mock_config.return_value = cfg

        from sources.pubmed import PubMedSource
        from utils.errors import DataSourceError

        source = PubMedSource()
        with pytest.raises(DataSourceError, match="email"):
            source.search("test query")

    @patch("sources.pubmed.get_config")
    @patch("sources.pubmed._get")
    def test_search_multiple_articles(self, mock_get, mock_config):
        mock_config.return_value = _make_pubmed_config()

        esearch_xml = """<?xml version="1.0"?>
        <eSearchResult>
            <Count>2</Count>
            <RetMax>2</RetMax>
            <WebEnv>ENV456</WebEnv>
            <QueryKey>2</QueryKey>
            <IdList><Id>111</Id><Id>222</Id></IdList>
        </eSearchResult>"""

        efetch_xml = """<?xml version="1.0"?>
        <PubmedArticleSet>
            <PubmedArticle>
                <MedlineCitation>
                    <PMID>111</PMID>
                    <Article>
                        <ArticleTitle>First Paper</ArticleTitle>
                        <AuthorList>
                            <Author><LastName>A</LastName><ForeName>B</ForeName></Author>
                        </AuthorList>
                        <Journal><Title>J1</Title><JournalIssue><PubDate><Year>2022</Year></PubDate></JournalIssue></Journal>
                    </Article>
                </MedlineCitation>
            </PubmedArticle>
            <PubmedArticle>
                <MedlineCitation>
                    <PMID>222</PMID>
                    <Article>
                        <ArticleTitle>Second Paper</ArticleTitle>
                        <AuthorList>
                            <Author><LastName>C</LastName><ForeName>D</ForeName></Author>
                        </AuthorList>
                        <Journal><Title>J2</Title><JournalIssue><PubDate><Year>2023</Year></PubDate></JournalIssue></Journal>
                    </Article>
                </MedlineCitation>
            </PubmedArticle>
        </PubmedArticleSet>"""

        esearch_resp = MagicMock()
        esearch_resp.content = esearch_xml.encode("utf-8")
        efetch_resp = MagicMock()
        efetch_resp.content = efetch_xml.encode("utf-8")
        mock_get.side_effect = [esearch_resp, efetch_resp]

        from sources.pubmed import PubMedSource

        source = PubMedSource()
        result = source.search("multi test", rows=2)

        assert result["total"] == 2
        assert len(result["results"]) == 2
        assert result["results"][0]["pmid"] == "111"
        assert result["results"][1]["pmid"] == "222"


# ===================================================================
# 3. arXiv tests
# ===================================================================


_ARXIV_ATOM_TEMPLATE = """<?xml version="1.0" encoding="UTF-8"?>
<feed xmlns="http://www.w3.org/2005/Atom"
      xmlns:arxiv="http://arxiv.org/schemas/atom">
  <title>ArXiv Query: search_query={query}</title>
  {entries}
</feed>"""

_ARXIV_ENTRY_TEMPLATE = """<entry>
    <id>http://arxiv.org/abs/{arxiv_id}v1</id>
    <title>{title}</title>
    <summary>{summary}</summary>
    {author_elements}
    <published>{published}</published>
    <arxiv:primary_category term="{category}" xmlns:arxiv="http://arxiv.org/schemas/atom"/>
    <link rel="alternate" type="text/html" href="http://arxiv.org/abs/{arxiv_id}v1"/>
    <link title="pdf" rel="related" type="application/pdf" href="http://arxiv.org/pdf/{arxiv_id}v1"/>
</entry>"""


def _build_arxiv_xml(query: str, entries_data: list[dict]) -> str:
    """Build a minimal arXiv Atom XML response."""
    entries_xml = []
    for e in entries_data:
        author_elements = "".join(
            f"<author><name>{a}</name></author>" for a in e.get("authors", [])
        )
        entries_xml.append(
            _ARXIV_ENTRY_TEMPLATE.format(
                arxiv_id=e.get("arxiv_id", "2401.00001"),
                title=e.get("title", "Untitled"),
                summary=e.get("summary", ""),
                author_elements=author_elements,
                published=e.get("published", "2024-01-01T00:00:00Z"),
                category=e.get("category", "cs.AI"),
            )
        )
    return _ARXIV_ATOM_TEMPLATE.format(
        query=query, entries="\n".join(entries_xml)
    )


class TestArxivSearch:
    """Test arXiv search with date filtering and ID normalization."""

    @patch("sources.arxiv.get_config")
    @patch.object(
        __import__("sources.arxiv", fromlist=["ArxivSource"]).ArxivSource,
        "_request",
    )
    def test_search_returns_unified_format(self, mock_request, mock_config):
        mock_config.return_value = _make_arxiv_config()

        xml = _build_arxiv_xml("transformer", [
            {
                "arxiv_id": "2401.12345",
                "title": "Attention Is All You Need Again",
                "summary": "We revisit the transformer architecture.",
                "authors": ["Alice Smith", "Bob Jones"],
                "published": "2024-01-22T00:00:00Z",
                "category": "cs.CL",
            }
        ])
        mock_request.return_value = xml

        from sources.arxiv import ArxivSource

        source = ArxivSource()
        result = source.search("transformer", rows=5)

        assert "total" in result
        assert "results" in result
        assert result["total"] == 1
        assert result["source"] == "arxiv"

        item = result["results"][0]
        assert item["title"] == "Attention Is All You Need Again"
        assert item["authors"] == ["Alice Smith", "Bob Jones"]
        assert item["year"] == 2024
        assert item["arxiv_id"] == "2401.12345"
        assert item["categories"] == ["cs.CL"]
        assert item["source"] == "arxiv"

    @patch("sources.arxiv.get_config")
    @patch.object(
        __import__("sources.arxiv", fromlist=["ArxivSource"]).ArxivSource,
        "_request",
    )
    def test_search_with_date_filter(self, mock_request, mock_config):
        mock_config.return_value = _make_arxiv_config()

        xml = _build_arxiv_xml("LLM", [])
        mock_request.return_value = xml

        from sources.arxiv import ArxivSource

        source = ArxivSource()
        source.search("LLM", rows=5, date_from="2024-01-01", date_to="2024-06-30")

        called_params = mock_request.call_args[0][0]
        search_query = called_params["search_query"]
        assert "submittedDate:[" in search_query
        assert "202401010000+TO+202406302359" in search_query

    @patch("sources.arxiv.get_config")
    @patch.object(
        __import__("sources.arxiv", fromlist=["ArxivSource"]).ArxivSource,
        "_request",
    )
    def test_search_with_categories(self, mock_request, mock_config):
        mock_config.return_value = _make_arxiv_config()

        xml = _build_arxiv_xml("robotics", [])
        mock_request.return_value = xml

        from sources.arxiv import ArxivSource

        source = ArxivSource()
        source.search("robotics", rows=5, categories=["cs.RO", "cs.AI"])

        called_params = mock_request.call_args[0][0]
        search_query = called_params["search_query"]
        assert "cat:cs.RO" in search_query
        assert "cat:cs.AI" in search_query

    @patch("sources.arxiv.get_config")
    @patch.object(
        __import__("sources.arxiv", fromlist=["ArxivSource"]).ArxivSource,
        "_request",
    )
    def test_get_by_id(self, mock_request, mock_config):
        mock_config.return_value = _make_arxiv_config()

        xml = _build_arxiv_xml("id", [
            {
                "arxiv_id": "2301.00001",
                "title": "Foundational LLM Paper",
                "summary": "We introduce a new LLM.",
                "authors": ["Researcher One"],
                "published": "2023-01-01T00:00:00Z",
                "category": "cs.AI",
            }
        ])
        mock_request.return_value = xml

        from sources.arxiv import ArxivSource

        source = ArxivSource()
        result = source.get_by_id("2301.00001")

        assert result["title"] == "Foundational LLM Paper"
        assert result["arxiv_id"] == "2301.00001"
        assert result["source"] == "arxiv"

    @patch("sources.arxiv.get_config")
    @patch.object(
        __import__("sources.arxiv", fromlist=["ArxivSource"]).ArxivSource,
        "_request",
    )
    def test_get_by_id_not_found(self, mock_request, mock_config):
        mock_config.return_value = _make_arxiv_config()

        # Empty feed = no results
        xml = _build_arxiv_xml("id", [])
        mock_request.return_value = xml

        from sources.arxiv import ArxivSource
        from utils.errors import DataSourceError

        source = ArxivSource()
        with pytest.raises(DataSourceError, match="not found"):
            source.get_by_id("9999.99999")

    def test_normalize_id_strips_url_prefix_and_version(self):
        from sources.arxiv import ArxivSource

        source = ArxivSource.__new__(ArxivSource)

        assert source._normalize_id("http://arxiv.org/abs/2401.12345v1") == "2401.12345"
        assert source._normalize_id("https://arxiv.org/abs/2401.12345v2") == "2401.12345"
        assert source._normalize_id("2401.12345") == "2401.12345"
        assert source._normalize_id("2401.12345v3") == "2401.12345"

    def test_build_date_filter_syntax(self):
        from sources.arxiv import ArxivSource

        result = ArxivSource._build_date_filter("2024-01-01", "2024-12-31")
        assert result == "submittedDate:[202401010000+TO+202412312359]"

    def test_build_date_filter_only_from(self):
        from sources.arxiv import ArxivSource

        result = ArxivSource._build_date_filter("2024-06-01", None)
        assert "submittedDate:[" in result
        assert "202406010000" in result
        assert "999912312359" in result

    def test_build_date_filter_empty(self):
        from sources.arxiv import ArxivSource

        assert ArxivSource._build_date_filter(None, None) == ""


# ===================================================================
# 4. ID auto-detection tests
# ===================================================================


class TestDetectIdType:
    """Test _detect_id_type auto-identification logic."""

    def test_detect_doi(self):
        from academic_search_server import _detect_id_type

        assert _detect_id_type("10.1038/nature12373") == "doi"
        assert _detect_id_type("10.1126/science.abc1234") == "doi"
        assert _detect_id_type("10.1016/j.cell.2023.01.001") == "doi"

    def test_detect_pmid(self):
        from academic_search_server import _detect_id_type

        assert _detect_id_type("12345678") == "pmid"
        assert _detect_id_type("1234567") == "pmid"

    def test_detect_arxiv(self):
        from academic_search_server import _detect_id_type

        assert _detect_id_type("2401.12345") == "arxiv"
        assert _detect_id_type("2301.00001") == "arxiv"
        assert _detect_id_type("2401.12345v1") == "arxiv"

    def test_detect_doi_with_whitespace(self):
        from academic_search_server import _detect_id_type

        assert _detect_id_type("  10.1038/nature12373  ") == "doi"

    def test_detect_unknown_raises(self):
        from academic_search_server import _detect_id_type

        with pytest.raises(ValueError, match="Cannot detect"):
            _detect_id_type("abc123")

    def test_detect_short_number_raises(self):
        """6-digit number is too short for PMID (needs 7-8)."""
        from academic_search_server import _detect_id_type

        with pytest.raises(ValueError, match="Cannot detect"):
            _detect_id_type("123456")


class TestResolveIdType:
    """Test _resolve_id_type explicit and auto modes."""

    def test_explicit_doi(self):
        from academic_search_server import _resolve_id_type

        assert _resolve_id_type("10.1038/test", "doi") == "doi"

    def test_explicit_pmid(self):
        from academic_search_server import _resolve_id_type

        assert _resolve_id_type("12345678", "pmid") == "pmid"

    def test_auto_delegates(self):
        from academic_search_server import _resolve_id_type

        assert _resolve_id_type("10.1038/test", "auto") == "doi"
        assert _resolve_id_type("12345678", "auto") == "pmid"
        assert _resolve_id_type("2401.12345", "auto") == "arxiv"

    def test_invalid_type_raises(self):
        from academic_search_server import _resolve_id_type

        with pytest.raises(ValueError, match="Unsupported"):
            _resolve_id_type("anything", "invalid_type")

"""Utility modules for academic search."""

from .config import Config, get_config
from .errors import AcademicSearchError, ConfigError, DataSourceError, TimeoutError
from .logging import setup_logging

__all__ = [
    "AcademicSearchError",
    "DataSourceError",
    "TimeoutError",
    "ConfigError",
    "setup_logging",
    "get_config",
    "Config",
]

"""Configuration management for academic search server."""

import os
from pathlib import Path

import toml


class Config:
    def __init__(self, config_path: str | Path | None = None):
        if config_path is None:
            config_path = Path(__file__).parent.parent / "config.toml"
        self._config = toml.load(config_path)

    @property
    def pubmed_email(self) -> str:
        return os.environ.get("PUBMED_EMAIL") or self._config.get("pubmed", {}).get("email", "")

    @property
    def pubmed_api_key(self) -> str:
        return os.environ.get("NCBI_API_KEY") or self._config.get("pubmed", {}).get("api_key", "")

    @property
    def crossref_mailto(self) -> str:
        return self._config.get("crossref", {}).get("mailto", "")

    @property
    def crossref_timeout(self) -> int:
        return self._config.get("crossref", {}).get("timeout", 15)

    @property
    def arxiv_timeout(self) -> int:
        return self._config.get("arxiv", {}).get("timeout", 30)

    @property
    def default_rows(self) -> int:
        return self._config.get("general", {}).get("default_rows", 5)

    @property
    def max_rows(self) -> int:
        return self._config.get("general", {}).get("max_rows", 50)


# Global config instance
_config: Config | None = None


def get_config() -> Config:
    global _config
    if _config is None:
        _config = Config()
    return _config

"""Unified error types for academic search operations."""


class AcademicSearchError(Exception):
    """Base exception for academic search operations."""


class DataSourceError(AcademicSearchError):
    """Error from a specific data source."""

    def __init__(self, source: str, message: str, original_error: Exception | None = None):
        self.source = source
        self.original_error = original_error
        super().__init__(f"[{source}] {message}")


class TimeoutError(AcademicSearchError):
    """Request timeout after retries."""


class ConfigError(AcademicSearchError):
    """Configuration error."""

"""Structured logging for academic search operations."""

import json
import logging
import sys
from datetime import datetime, timezone


class JSONFormatter(logging.Formatter):
    def format(self, record):
        log_data = {
            "timestamp": datetime.now(timezone.utc).isoformat(),
            "level": record.levelname,
            "tool": getattr(record, "tool", None),
            "query": getattr(record, "query", None),
            "sources": getattr(record, "sources", None),
            "duration_ms": getattr(record, "duration_ms", None),
            "results_count": getattr(record, "results_count", None),
            "message": record.getMessage(),
        }
        if record.exc_info:
            log_data["exception"] = self.formatException(record.exc_info)
        return json.dumps({k: v for k, v in log_data.items() if v is not None})


def setup_logging(level: str = "INFO") -> logging.Logger:
    logger = logging.getLogger("academic-search")
    logger.setLevel(getattr(logging, level.upper()))
    handler = logging.StreamHandler(sys.stderr)
    handler.setFormatter(JSONFormatter())
    logger.addHandler(handler)
    return logger

# `nature-academic-search` Skill

[中文说明](README.md)

`nature-academic-search` supports multi-source scholarly search, bibliographic metadata checks, citation formatting, strict external-citation audits, and influential-citer analysis.

## What To Use It For

- Search papers across CrossRef, PubMed, arXiv, Scopus, ScienceDirect, and related scholarly sources.
- Retrieve structured paper details by DOI, PMID, or arXiv ID.
- Generate Nature, APA, IEEE, Vancouver, and other citation styles.
- Build MeSH-assisted PubMed search strategies for biomedical topics.
- Audit strict external citations for a target paper while excluding self-citations, team citations, and obvious collaboration-network citations.
- Identify how academy members, Fellows, highly cited scholars, institutional leaders, or field experts cited a target paper.

## Typical Requests

- "Check the strict external citations for this paper and list influential citers."
- "Find 20 recent Nature/Science/Cell-related papers on this topic."
- "Convert these DOIs into Nature-style references and export RIS."
- "Add MeSH terms and synonyms to this PubMed query."

## What You Need To Provide

- Search topic, keywords, time range, journal scope, or target DOI.
- Whether Scopus / ScienceDirect or other locally configured sources may be used.
- The exclusion rule for strict external citations, such as whether to exclude same-institution, coauthor, or lab-network citations.

## Outputs

- Deduplicated paper table with title, authors, year, journal, DOI, and source.
- Formatted citation text or `.ris`, `.bib`, `.nbib`, `.enw` reference-management files.
- Strict external-citation table and citation-context summary.
- Search strategy, source notes, and records that could not be verified.

## Runtime and Dependencies

- PubMed search needs `PUBMED_EMAIL` or `mcp-server/config.toml`.
- Scopus / ScienceDirect depend on local `pybliometrics` configuration, normally read from `~/.config/pybliometrics.cfg`.
- Do not write Elsevier API keys into repository files; keep them in local secure configuration.

## Boundaries

- Secondary scholarly indexes are discovery aids; key fields should be verified through DOI records, PubMed, publisher pages, or authoritative databases.
- Strict external-citation analysis needs a clear exclusion rule; missing author or affiliation metadata is marked as uncertain.
- Restricted databases are not bypassed; the skill reports missing sources or asks for user login when needed.

## Related Skills

- `nature-citation`: match Nature/CNS/Cell supporting citations for manuscript claims.
- `nature-literature-pipeline`: turn one-off search into ongoing literature monitoring.
- `nature-ref-verifier`: verify reference-list metadata field by field.

# `nature-academic-search` 技能

[English](README_EN.md)

`nature-academic-search` 用于多源学术检索、文献元数据核查、引用格式生成、严格他引审计和高影响力引用者分析。

## 适合用它做什么

- 在 CrossRef、PubMed、arXiv、Scopus、ScienceDirect 等来源中检索论文。
- 根据 DOI、PMID 或 arXiv ID 获取结构化文献信息。
- 生成 Nature、APA、IEEE、Vancouver 等引用格式。
- 构建 MeSH 检索式，辅助医学和生命科学主题检索。
- 审计目标论文的严格他引，排除自引、团队引和明显合作网络引用。
- 识别院士、Fellow、高被引学者、院校领导或领域专家如何引用目标论文。

## 典型请求

- “帮我查这篇论文的严格他引，并列出高影响力引用者。”
- “按这个主题找 20 篇近五年的 Nature/Science/Cell 相关论文。”
- “把这些 DOI 转成 Nature 格式引用，并导出 RIS。”
- “为 PubMed 检索式补 MeSH 词和同义词。”

## 你需要提供

- 检索主题、关键词、时间范围、期刊范围或目标文献 DOI。
- 是否允许使用 Scopus / ScienceDirect 等需要本机配置的来源。
- 严格他引的排除口径，例如是否排除同机构、共同作者或课题组网络。

## 产出

- 去重后的文献表，包含标题、作者、年份、期刊、DOI 和来源。
- 引用格式文本或 `.ris`、`.bib`、`.nbib`、`.enw` 等文献管理文件。
- 严格他引统计表和引用上下文摘要。
- 查询策略、数据源说明和无法验证的条目列表。

## 运行和依赖

- PubMed 检索需要配置 `PUBMED_EMAIL` 或 `mcp-server/config.toml`。
- Scopus / ScienceDirect 依赖本机 `pybliometrics` 配置，默认读取 `~/.config/pybliometrics.cfg`。
- 不要把 Elsevier API key 写入仓库文件；只在本机安全配置中保存。

## 边界

- 二级学术索引只能作为发现线索，关键字段应回到 DOI、PubMed、出版社页面或数据库记录核实。
- 严格他引判断需要明确排除规则；没有作者/机构信息时会标注不确定。
- 无权限数据库不会被绕过，只会报告缺失来源或需要用户登录。

## 相关技能

- `nature-citation`：为手稿 claim 匹配 Nature/CNS/Cell 支撑文献。
- `nature-literature-pipeline`：把一次性检索升级为持续文献监测。
- `nature-ref-verifier`：逐条核查参考文献字段。

# Citation Parser

Strategy reference for extracting citations from documents. Used by Workflow 2 (Citation Verification).

## Extraction by Source Format

### .docx

**Method:** Use python-docx to iterate paragraphs. For each paragraph, apply regex patterns below in order.

**Preprocessing:**
- Skip paragraphs inside EndNote field codes: if paragraph text matches `ADDIN EN\.\w+`, skip entire paragraph.
- Skip Zotero field codes: if paragraph text matches `\{[|]?\|[^}]+\}`, skip.
- Strip inline Zotero markers: remove `\{citation:\d+\}` spans from text.

**Regex patterns** (apply to cleaned text):

| Pattern | Target | Example |
|---------|--------|---------|
| `10\.\d{4,}/[^\s]+` | DOI | `10.1038/nature14539` |
| `PMID:?\s*\d{7,8}` | PMID | `PMID: 28344011` |
| `PMCID:?\s*PMC\d+` | PMCID | `PMCID: PMC5390525` |
| `arXiv:\s*\d{4}\.\d{4,}(v\d+)?` | arXiv ID | `arXiv: 1706.03762v7` |

### .tex / .bib

**Method:** Read file as text. Apply patterns:

| Pattern | Target |
|---------|--------|
| `\\cite\{([^}]+)\}` | Citation keys (comma-separated) |
| `\\bibitem\{([^}]+)\}` | Bibitem keys |
| `@article\{([^,]+),` | BibTeX entry keys |
| `doi\s*=\s*\{([^}]+)\}` | DOI in BibTeX |
| `pmid\s*=\s*\{([^}]+)\}` | PMID in BibTeX |

**Resolution:** Citation keys → resolve via .bib file `@article` entries. If standalone .tex without .bib, flag as `manual_needed`.

**natbib/biblatex:** `\\citep`, `\\citet`, `\\textcite` all reduce to `\\cite` semantics for key extraction.

### .txt

**Method:** Line-by-line scan. Apply the same DOI/PMID/arXiv regex patterns as .docx.

**Edge cases:**
- Paste-imported references may have line-wrapping that splits DOIs. Before scanning, join lines where the previous line ends mid-DOI pattern (no terminal punctuation, next line starts with alphanumeric).
- References without identifiers: extract first line as candidate title, mark for title+author resolution via MCP search.

## Resolution Priority

For each extracted reference:
1. DOI → `get_paper_by_doi` or `search_crossref`
2. PMID → `pubmed_fetch_articles`
3. arXiv ID → `search_arxiv`
4. Title + first author → `pubmed_search_articles` or `search_crossref`
5. Citation key only (unresolvable from .bib) → `manual_needed`

## Classification Labels

| Label | Condition |
|-------|-----------|
| `verified` | Retrieved metadata matches document metadata (title + journal + year all match) |
| `mismatch` | Retrieved metadata exists but conflicts with document |
| `not_found` | No match found in any database |
| `suspicious` | Match found but metadata incomplete (e.g., missing DOI, only partial title match) |
| `manual_needed` | Cannot resolve to a database query (no DOI/PMID, title too generic) |

# Dedup Engine

Unified deduplication logic shared by all workflows that merge literature result lists
(Workflow 1 multi-source search, Workflow 2 citation verification, Workflow 5a related papers).

## Primary Key: DOI

1. Extract DOI from each record. Match `10.\d{4,}/[^\s]+` pattern.
2. Strip leading `https://doi.org/` prefix if present.
3. Normalize: lowercase, trim whitespace.
4. Records sharing a normalized DOI are duplicates.

## Fallback Key: Title + First Author

When DOI is missing from either record:

1. **Normalize titles:**
   - Lowercase
   - Remove punctuation (.,;:!?()[]"")
   - Remove English stopwords (a, an, the, in, of, for, on, to, and, with, by, et, al)
   - Collapse multiple whitespace to single space
   - Strip leading/trailing whitespace
2. **Tokenize:** split normalized titles into word tokens.
3. **Compute Jaccard similarity:**
   - `intersection = set(tokens_A) & set(tokens_B)`
   - `union = set(tokens_A) | set(tokens_B)`
   - `similarity = len(intersection) / len(union)` if union non-empty, else 0
4. **Compare first-author surnames:**
   - Extract surname: take the first author string, split on commas, take first token, lowercase, strip.
   - Two records match if surnames are identical AND Jaccard similarity >= 0.90.

## Merge Preference

When a duplicate pair spans sources, prefer the record with (in order):
1. More complete metadata (DOI + volume + pages all present)
2. Publisher source over preprint source
3. Higher citation count as tiebreaker

## Usage in Workflows

- **Workflow 1 (Multi-source search):** After parallel MCP search, run dedup on merged result list before ranking/presentation.
- **Workflow 2 (Citation verification):** When a document reference resolves to multiple candidate matches, use dedup to collapse identical candidates before classification.
- **Workflow 5a (Related papers):** When related-paper results overlap with the source search, dedup before presenting.

@article{pmid28344011,
  author  = {Dheda K and Gumbo T and Maartens G and Dooley KE and McNerney R and Murray M and Furin J and Nardell EA and London L and Lessem E and Theron G and van Helden P and Niemann S and Merker M and Dowdy D and Van Rie A and Siu GK and Pasipanodya JG and Rodrigues C and Clark TG and Sirgel FA and Esmail A and Lin HH and Atre SR and Schaaf HS and Chang KC and Lange C and Nahid P and Udwadia ZF and Horsburgh CR Jr and Churchyard GJ and Menzies D and Hesseling AC and Nuermberger E and McIlleron H and Fennelly KP and Goemaere E and Jaramillo E and Low M and Jara CM and Padayatchi N and Warren RM},
  title   = {{The epidemiology, pathogenesis, transmission, diagnosis, and management of multidrug-resistant, extensively drug-resistant, and incurable tuberculosis.}},
  journal = {The Lancet. Respiratory medicine},
  year    = {2017},
  abstract = {Global tuberculosis incidence has declined marginally over the past decade, and tuberculosis remains out of control in several parts of the world including Africa and Asia. Although tuberculosis control has been effective in some regions of the world, these gains are threatened by the increasing burden of multidrug-resistant (MDR) and extensively drug-resistant (XDR) tuberculosis. XDR tuberculosis has evolved in several tuberculosis-endemic countries to drug-incurable or programmatically incurable tuberculosis (totally drug-resistant tuberculosis). This poses several challenges similar to those encountered in the pre-chemotherapy era, including the inability to cure tuberculosis, high mortality, and the need for alternative methods to prevent disease transmission. This phenomenon mirrors the worldwide increase in antimicrobial resistance and the emergence of other MDR pathogens, such as malaria, HIV, and Gram-negative bacteria. MDR and XDR tuberculosis are associated with high morbidity and substantial mortality, are a threat to health-care workers, prohibitively expensive to treat, and are therefore a serious public health problem. In this Commission, we examine several aspects of drug-resistant tuberculosis. The traditional view that acquired resistance to antituberculous drugs is driven by poor compliance and programmatic failure is now being questioned, and several lines of evidence suggest that alternative mechanisms-including pharmacokinetic variability, induction of efflux pumps that transport the drug out of cells, and suboptimal drug penetration into tuberculosis lesions-are likely crucial to the pathogenesis of drug-resistant tuberculosis. These factors have implications for the design of new interventions, drug delivery and dosing mechanisms, and public health policy. We discuss epidemiology and transmission dynamics, including new insights into the fundamental biology of transmission, and we review the utility of newer diagnostic tools, including molecular tests and next-generation whole-genome sequencing, and their potential for clinical effectiveness. Relevant research priorities are highlighted, including optimal medical and surgical management, the role of newer and repurposed drugs (including bedaquiline, delamanid, and linezolid), pharmacokinetic and pharmacodynamic considerations, preventive strategies (such as prophylaxis in MDR and XDR contacts), palliative and patient-orientated care aspects, and medicolegal and ethical issues.},
  pmid    = {28344011},
}

PMID- 28344011
OWN - NLM
STAT- Publisher
LR  - 20250530
IS  - 2213-2619 (Electronic)
IS  - 2213-2600 (Linking)
DP  - 2017 Mar 15
TI  - The epidemiology, pathogenesis, transmission, diagnosis, and management of 
      multidrug-resistant, extensively drug-resistant, and incurable tuberculosis.
LID - S2213-2600(17)30079-6 [pii]
LID - 10.1016/S2213-2600(17)30079-6 [doi]
AB  - Global tuberculosis incidence has declined marginally over the past decade, and 
      tuberculosis remains out of control in several parts of the world including 
      Africa and Asia. Although tuberculosis control has been effective in some regions 
      of the world, these gains are threatened by the increasing burden of 
      multidrug-resistant (MDR) and extensively drug-resistant (XDR) tuberculosis. XDR 
      tuberculosis has evolved in several tuberculosis-endemic countries to 
      drug-incurable or programmatically incurable tuberculosis (totally drug-resistant 
      tuberculosis). This poses several challenges similar to those encountered in the 
      pre-chemotherapy era, including the inability to cure tuberculosis, high 
      mortality, and the need for alternative methods to prevent disease transmission. 
      This phenomenon mirrors the worldwide increase in antimicrobial resistance and 
      the emergence of other MDR pathogens, such as malaria, HIV, and Gram-negative 
      bacteria. MDR and XDR tuberculosis are associated with high morbidity and 
      substantial mortality, are a threat to health-care workers, prohibitively 
      expensive to treat, and are therefore a serious public health problem. In this 
      Commission, we examine several aspects of drug-resistant tuberculosis. The 
      traditional view that acquired resistance to antituberculous drugs is driven by 
      poor compliance and programmatic failure is now being questioned, and several 
      lines of evidence suggest that alternative mechanisms-including pharmacokinetic 
      variability, induction of efflux pumps that transport the drug out of cells, and 
      suboptimal drug penetration into tuberculosis lesions-are likely crucial to the 
      pathogenesis of drug-resistant tuberculosis. These factors have implications for 
      the design of new interventions, drug delivery and dosing mechanisms, and public 
      health policy. We discuss epidemiology and transmission dynamics, including new 
      insights into the fundamental biology of transmission, and we review the utility 
      of newer diagnostic tools, including molecular tests and next-generation 
      whole-genome sequencing, and their potential for clinical effectiveness. Relevant 
      research priorities are highlighted, including optimal medical and surgical 
      management, the role of newer and repurposed drugs (including bedaquiline, 
      delamanid, and linezolid), pharmacokinetic and pharmacodynamic considerations, 
      preventive strategies (such as prophylaxis in MDR and XDR contacts), palliative 
      and patient-orientated care aspects, and medicolegal and ethical issues.
CI  - Copyright (c) 2017 Elsevier Ltd. All rights reserved.
FAU - Dheda, Keertan
AU  - Dheda K
AD  - Lung Infection and Immunity Unit, Department of Medicine, Division of Pulmonology 
      and UCT Lung Institute, University of Cape Town, Groote Schuur Hospital, Cape 
      Town, South Africa. Electronic address: [email protected].
FAU - Gumbo, Tawanda
AU  - Gumbo T
AD  - Center for Infectious Diseases Research and Experimental Therapeutics, Baylor 
      Research Institute, Baylor University Medical Center, Dallas, TX, USA.
FAU - Maartens, Gary
AU  - Maartens G
AD  - Division of Clinical Pharmacology, Department of Medicine, University of Cape 
      Town, Cape Town, South Africa.
FAU - Dooley, Kelly E
AU  - Dooley KE
AD  - Center for Tuberculosis Research, Johns Hopkins University School of Medicine, 
      Baltimore, MD, USA.
FAU - McNerney, Ruth
AU  - McNerney R
AD  - Lung Infection and Immunity Unit, Department of Medicine, Division of Pulmonology 
      and UCT Lung Institute, University of Cape Town, Groote Schuur Hospital, Cape 
      Town, South Africa.
FAU - Murray, Megan
AU  - Murray M
AD  - Department of Global Health and Social Medicine, Department of Medicine, Harvard 
      Medical School, Boston, MA, USA.
FAU - Furin, Jennifer
AU  - Furin J
AD  - Department of Global Health and Social Medicine, Department of Medicine, Harvard 
      Medical School, Boston, MA, USA.
FAU - Nardell, Edward A
AU  - Nardell EA
AD  - TH Chan School of Public Health, Department of Medicine, Harvard Medical School, 
      Boston, MA, USA.
FAU - London, Leslie
AU  - London L
AD  - School of Public Health and Medicine, University of Cape Town, Cape Town, South 
      Africa.
FAU - Lessem, Erica
AU  - Lessem E
AD  - Treatment Action Group, New York, NY, USA.
FAU - Theron, Grant
AU  - Theron G
AD  - SA MRC Centre for Tuberculosis Research/DST/NRF Centre of Excellence for 
      Biomedical Tuberculosis Research, Division of Molecular Biology and Human 
      Genetics, Stellenbosch University, Tygerberg, South Africa.
FAU - van Helden, Paul
AU  - van Helden P
AD  - SA MRC Centre for Tuberculosis Research/DST/NRF Centre of Excellence for 
      Biomedical Tuberculosis Research, Division of Molecular Biology and Human 
      Genetics, Stellenbosch University, Tygerberg, South Africa.
FAU - Niemann, Stefan
AU  - Niemann S
AD  - Molecular and Experimental Mycobacteriology, Research Center Borstel, Borstel, 
      Schleswig-Holstein, Germany; German Centre for Infection Research (DZIF), Partner 
      Site Borstel, Borstel, Schleswig-Holstein, Germany.
FAU - Merker, Matthias
AU  - Merker M
AD  - Molecular and Experimental Mycobacteriology, Research Center Borstel, Borstel, 
      Schleswig-Holstein, Germany.
FAU - Dowdy, David
AU  - Dowdy D
AD  - Department of Epidemiology, Johns Hopkins Bloomberg School of Public Health, 
      Baltimore, MD, USA.
FAU - Van Rie, Annelies
AU  - Van Rie A
AD  - University of North Carolina at Chapel Hill, Chapel Hill, NC, USA; International 
      Health Unit, Epidemiology and Social Medicine, Faculty of Medicine, University of 
      Antwerp, Antwerp, Belgium.
FAU - Siu, Gilman K H
AU  - Siu GK
AD  - Department of Health Technology and Informatics, The Hong Kong Polytechnic 
      University, Hung Hom, Hong Kong SAR, China.
FAU - Pasipanodya, Jotam G
AU  - Pasipanodya JG
AD  - Center for Infectious Diseases Research and Experimental Therapeutics, Baylor 
      Research Institute, Baylor University Medical Center, Dallas, TX, USA.
FAU - Rodrigues, Camilla
AU  - Rodrigues C
AD  - Department of Microbiology, P.D. Hinduja National Hospital & Medical Research 
      Centre, Mumbai, India.
FAU - Clark, Taane G
AU  - Clark TG
AD  - Faculty of Infectious and Tropical Diseases and Faculty of Epidemiology and 
      Population Health, London School of Hygiene & Tropical Medicine, London, UK.
FAU - Sirgel, Frik A
AU  - Sirgel FA
AD  - SA MRC Centre for Tuberculosis Research/DST/NRF Centre of Excellence for 
      Biomedical Tuberculosis Research, Division of Molecular Biology and Human 
      Genetics, Stellenbosch University, Tygerberg, South Africa.
FAU - Esmail, Aliasgar
AU  - Esmail A
AD  - Lung Infection and Immunity Unit, Department of Medicine, Division of Pulmonology 
      and UCT Lung Institute, University of Cape Town, Groote Schuur Hospital, Cape 
      Town, South Africa.
FAU - Lin, Hsien-Ho
AU  - Lin HH
AD  - Institute of Epidemiology and Preventive Medicine, National Taiwan University, 
      Taipei, Taiwan.
FAU - Atre, Sachin R
AU  - Atre SR
AD  - Center for Clinical Global Health Education (CCGHE), Johns Hopkins University, 
      Baltimore, MD, USA; Medical College, Hospital and Research Centre, Pimpri, Pune, 
      India.
FAU - Schaaf, H Simon
AU  - Schaaf HS
AD  - Desmond Tutu TB Centre, Department of Paediatrics and Child Health, Faculty of 
      Medicine and Health Sciences, Stellenbosch University, Cape Town, South Africa.
FAU - Chang, Kwok Chiu
AU  - Chang KC
AD  - Tuberculosis and Chest Service, Centre for Health Protection, Department of 
      Health, Hong Kong SAR, China.
FAU - Lange, Christoph
AU  - Lange C
AD  - Division of Clinical Infectious Diseases, German Center for Infection Research, 
      Research Center Borstel, Borstel, Schleswig-Holstein, Germany; International 
      Health/Infectious Diseases, University of Lubeck, Lubeck, Germany; Department of 
      Medicine, Karolinska Institute, Stockholm, Sweden; Department of Medicine, 
      University of Namibia School of Medicine, Windhoek, Namibia.
FAU - Nahid, Payam
AU  - Nahid P
AD  - Division of Pulmonary and Critical Care, San Francisco General Hospital, 
      University of California, San Francisco, CA, USA.
FAU - Udwadia, Zarir F
AU  - Udwadia ZF
AD  - Pulmonary Department, Hinduja Hospital & Research Center, Mumbai, India.
FAU - Horsburgh, C Robert Jr
AU  - Horsburgh CR Jr
AD  - Schools of Public Health Medicine, Boston University, Boston, MA, USA.
FAU - Churchyard, Gavin J
AU  - Churchyard GJ
AD  - Aurum Institute, Johannesburg, South Africa; School of Public Health, University 
      of Witwatersrand, Johannesburg, South Africa; Advancing Treatment and Care for 
      TB/HIV, South African Medical Research Council, Johannesburg, South Africa.
FAU - Menzies, Dick
AU  - Menzies D
AD  - Montreal Chest Institute, McGill University, Montreal, QC, Canada.
FAU - Hesseling, Anneke C
AU  - Hesseling AC
AD  - Desmond Tutu TB Centre, Department of Paediatrics and Child Health, Faculty of 
      Medicine and Health Sciences, Stellenbosch University, Cape Town, South Africa.
FAU - Nuermberger, Eric
AU  - Nuermberger E
AD  - Center for Tuberculosis Research, Johns Hopkins University School of Medicine, 
      Baltimore, MD, USA.
FAU - McIlleron, Helen
AU  - McIlleron H
AD  - Division of Clinical Pharmacology, Department of Medicine, University of Cape 
      Town, Cape Town, South Africa.
FAU - Fennelly, Kevin P
AU  - Fennelly KP
AD  - Pulmonary Clinical Medicine Section, Division of Intramural Research, National 
      Heart, Lung, and Blood Institute (NHLBI), National Institutes of Health (NIH), 
      Bethesda, MD, USA.
FAU - Goemaere, Eric
AU  - Goemaere E
AD  - MSF South Africa, Cape Town, South Africa; School of Public Health and Family 
      Medicine, University of Cape Town, Cape Town, South Africa.
FAU - Jaramillo, Ernesto
AU  - Jaramillo E
AD  - World Health Organization, Geneva, Switzerland.
FAU - Low, Marcus
AU  - Low M
AD  - Treatment Action Campaign, Johannesburg, South Africa.
FAU - Jara, Carolina Moran
AU  - Jara CM
AD  - Socios en Salud, Lima, Peru.
FAU - Padayatchi, Nesri
AU  - Padayatchi N
AD  - Centre for the AIDS Programme of Research in South Africa (CAPRISA), MRC HIV-TB 
      Pathogenesis and Treatment Research Unit, Durban, South Africa.
FAU - Warren, Robin M
AU  - Warren RM
AD  - SA MRC Centre for Tuberculosis Research/DST/NRF Centre of Excellence for 
      Biomedical Tuberculosis Research, Division of Molecular Biology and Human 
      Genetics, Stellenbosch University, Tygerberg, South Africa.
LA  - eng
GR  - 001/WHO_/World Health Organization/International
GR  - MR/M007340/1/MRC_/Medical Research Council/United Kingdom
GR  - MR/N010469/1/MRC_/Medical Research Council/United Kingdom
PT  - Journal Article
PT  - Review
DEP - 20170315
PL  - England
TA  - Lancet Respir Med
JT  - The Lancet. Respiratory medicine
JID - 101605555
EDAT- 2017/03/28 06:00
MHDA- 2017/03/28 06:00
CRDT- 2017/03/28 06:00
PHST- 2016/07/15 00:00 [received]
PHST- 2016/10/24 00:00 [revised]
PHST- 2016/12/08 00:00 [accepted]
PHST- 2017/03/28 06:00 [entrez]
PHST- 2017/03/28 06:00 [pubmed]
PHST- 2017/03/28 06:00 [medline]
AID - S2213-2600(17)30079-6 [pii]
AID - 10.1016/S2213-2600(17)30079-6 [doi]
PST - aheadofprint
SO  - Lancet Respir Med. 2017 Mar 15:S2213-2600(17)30079-6. doi: 
      10.1016/S2213-2600(17)30079-6.

TY  - JOUR
AU  - Dheda K
AU  - Gumbo T
AU  - Maartens G
AU  - Dooley KE
AU  - McNerney R
AU  - Murray M
AU  - Furin J
AU  - Nardell EA
AU  - London L
AU  - Lessem E
AU  - Theron G
AU  - van Helden P
AU  - Niemann S
AU  - Merker M
AU  - Dowdy D
AU  - Van Rie A
AU  - Siu GK
AU  - Pasipanodya JG
AU  - Rodrigues C
AU  - Clark TG
AU  - Sirgel FA
AU  - Esmail A
AU  - Lin HH
AU  - Atre SR
AU  - Schaaf HS
AU  - Chang KC
AU  - Lange C
AU  - Nahid P
AU  - Udwadia ZF
AU  - Horsburgh CR Jr
AU  - Churchyard GJ
AU  - Menzies D
AU  - Hesseling AC
AU  - Nuermberger E
AU  - McIlleron H
AU  - Fennelly KP
AU  - Goemaere E
AU  - Jaramillo E
AU  - Low M
AU  - Jara CM
AU  - Padayatchi N
AU  - Warren RM
TI  - The epidemiology, pathogenesis, transmission, diagnosis, and management of multidrug-resistant, extensively drug-resistant, and incurable tuberculosis.
JO  - The Lancet. Respiratory medicine
JA  - Lancet Respir Med
PY  - 2017
N2  - Global tuberculosis incidence has declined marginally over the past decade, and tuberculosis remains out of control in several parts of the world including Africa and Asia. Although tuberculosis control has been effective in some regions of the world, these gains are threatened by the increasing burden of multidrug-resistant (MDR) and extensively drug-resistant (XDR) tuberculosis. XDR tuberculosis has evolved in several tuberculosis-endemic countries to drug-incurable or programmatically incurable tuberculosis (totally drug-resistant tuberculosis). This poses several challenges similar to those encountered in the pre-chemotherapy era, including the inability to cure tuberculosis, high mortality, and the need for alternative methods to prevent disease transmission. This phenomenon mirrors the worldwide increase in antimicrobial resistance and the emergence of other MDR pathogens, such as malaria, HIV, and Gram-negative bacteria. MDR and XDR tuberculosis are associated with high morbidity and substantial mortality, are a threat to health-care workers, prohibitively expensive to treat, and are therefore a serious public health problem. In this Commission, we examine several aspects of drug-resistant tuberculosis. The traditional view that acquired resistance to antituberculous drugs is driven by poor compliance and programmatic failure is now being questioned, and several lines of evidence suggest that alternative mechanisms-including pharmacokinetic variability, induction of efflux pumps that transport the drug out of cells, and suboptimal drug penetration into tuberculosis lesions-are likely crucial to the pathogenesis of drug-resistant tuberculosis. These factors have implications for the design of new interventions, drug delivery and dosing mechanisms, and public health policy. We discuss epidemiology and transmission dynamics, including new insights into the fundamental biology of transmission, and we review the utility of newer diagnostic tools, including molecular tests and next-generation whole-genome sequencing, and their potential for clinical effectiveness. Relevant research priorities are highlighted, including optimal medical and surgical management, the role of newer and repurposed drugs (including bedaquiline, delamanid, and linezolid), pharmacokinetic and pharmacodynamic considerations, preventive strategies (such as prophylaxis in MDR and XDR contacts), palliative and patient-orientated care aspects, and medicolegal and ethical issues.
AN  - PMID:28344011
DB  - PubMed
ER  - 

# RIS and BibTeX Format Specifications

## Contents

- [RIS Format](#ris-format)
- [BibTeX Format](#bibtex-format)
- [ENW (EndNote Tagged) Format](#enw-endnote-tagged-format)
- [Text Escaping (ris_escape)](#text-escaping-ris_escape)
- [Format Selection Guide](#format-selection-guide)


## RIS Format

RIS is the standard import format for EndNote, Zotero, and most reference managers.

### Journal article template
```
TY  - JOUR
AU  - Last, First
AU  - Last2, First2
TI  - Article Title
JO  - Journal Name (full)
JA  - Journal Abbreviation
PY  - 2024
VL  - 10
IS  - 3
SP  - 123
EP  - 145
DO  - 10.1234/example
UR  - https://doi.org/10.1234/example
AB  - Abstract text
KW  - keyword1
KW  - keyword2
AN  - PMID:12345678
DB  - PubMed
ER  -
```

### MEDLINE-to-RIS field mapping

| MEDLINE tag | RIS tag | Notes |
|---|---|---|
| `PMID-` | `AN  - PMID:` | Accession number |
| `TI  -` | `TI  -` | Title |
| `AU  -` | `AU  -` | Last, First format |
| `JT  -` | `JO  -` | Full journal title |
| `TA  -` | `JA  -` | Journal abbreviation |
| `DP  -` | `PY  -` | Extract year only |
| `VI  -` | `VL  -` | Volume |
| `IP  -` | `IS  -` | Issue |
| `PG  -` | `SP  -` / `EP  -` | Split on `-` |
| `AB  -` | `AB  -` | Abstract |
| `MH  -` | `KW  -` | MeSH as keywords |
| `LID -` | `DO  -` | DOI (strip `[doi]` suffix) |
| `AID -` | `DO  -` | Alternative DOI field |
| `SO  -` | `JO  -` | Source (journal + date + vol + pages) |

### RIS record separator
Each record must end with `ER  -` followed by a blank line.

## BibTeX Format

### Journal article template
```bibtex
@article{pmid12345678,
  author  = {Last, First and Last2, First2},
  title   = {Article Title},
  journal = {Journal Name},
  year    = {2024},
  volume  = {10},
  number  = {3},
  pages   = {123--145},
  doi     = {10.1234/example},
  url     = {https://doi.org/10.1234/example},
  abstract = {Abstract text},
  pmid    = {12345678}
}
```

### Citation key convention
- Use `pmid` prefix + PMID: `pmid12345678`
- If no PMID: use `firstauthorYEAR` format, lowercase: `smith2024`
- If duplicate keys: append `a`, `b`: `smith2024a`

### MEDLINE-to-BibTeX field mapping

| MEDLINE tag | BibTeX field | Notes |
|---|---|---|
| `PMID-` | `pmid` | Custom field |
| `TI  -` | `title` | Wrap in `{...}` to preserve case |
| `AU  -` | `author` | `Last, First and Last, First` |
| `JT  -` | `journal` | Full journal title |
| `DP  -` | `year` | Extract 4-digit year |
| `VI  -` | `volume` | |
| `IP  -` | `number` | |
| `PG  -` | `pages` | Replace `-` with `--` |
| `AB  -` | `abstract` | |
| `LID -` / `AID -` | `doi` | Strip `[doi]` suffix |

### Required fields by entry type

**`@article`**: author, title, journal, year
**`@book`**: author/editor, title, publisher, year
**`@inproceedings`**: author, title, booktitle, year
**`@phdthesis`**: author, title, school, year

### BibTeX cleaning rules
- Remove unescaped `&` in author fields (replace with `\&`)
- Wrap titles with special characters in `{...}`
- Strip HTML tags from abstract
- Ensure consistent author separator: ` and `
- Sort entries alphabetically by citation key

## ENW (EndNote Tagged) Format

ENW uses percent-tagged fields for EndNote import. Supported by EndNote desktop and EndNote Web.

### Journal article template
```
%0 Journal Article
%T Article Title
%A Last, First
%A Last2, First2
%J Journal Name
%V 10
%N 3
%P 123-145
%D 2024
%R 10.1234/example
%U https://doi.org/10.1234/example
%X Abstract text
```

### MEDLINE-to-ENW field mapping

| MEDLINE tag | ENW tag | Notes |
|---|---|---|
| `TI  -` | `%T` | Title |
| `AU  -` | `%A` | Last, First format; one per author |
| `JT  -` | `%J` | Full journal title |
| `VI  -` | `%V` | Volume |
| `IP  -` | `%N` | Issue |
| `PG  -` | `%P` | Pages |
| `DP  -` | `%D` | Year only |
| `LID -` / `AID -` | `%R` | DOI |
| — | `%U` | URL (derived from DOI) |
| `AB  -` | `%X` | Abstract |

### EndNote import instruction

In EndNote: File > Import > File, choose the `.enw` file, set Import Option to EndNote generated XML or Tagged format, then import.

## Text Escaping (ris_escape)

Applied to all free-text fields (TI, AU, JO, N2/AB, KW) in RIS and ENW output:

- HTML tags stripped: `<sup>1</sup>` → `1`
- Whitespace normalized: multiple spaces/newlines → single space
- Leading/trailing whitespace trimmed
- Abstract truncated to 500 chars in RIS/ENW to keep records compact

## Format Selection Guide

| User says | Use format |
|---|---|
| "EndNote", ".enw" | ENW (EndNote tagged) or RIS |
| "Zotero" | RIS (Zotero imports RIS natively) |
| "LaTeX", "BibTeX", "Bib" | BibTeX (.bib) |
| "PubMed format", "MEDLINE", ".nbib" | .nbib (default) |

# Search Strategy Guide

## Query Construction

### From topic to query
1. Extract core concepts from the research question
2. Identify synonyms and alternate spellings for each concept
3. For biomedical topics: map concepts to MeSH terms via `pubmed_lookup_mesh`
4. Assemble Boolean query: `(concept1 OR synonym1) AND (concept2 OR synonym2)`
5. Add field qualifiers for precision: `[Title/Abstract]`, `[MeSH Terms]`, `[Journal]`
6. Test and refine — if >500 results, add filters; if <10, broaden terms

### Query templates by domain

| Domain | Template |
|--------|----------|
| Medical | `("disease"[MeSH] OR "disease"[tiab]) AND ("treatment"[MeSH] OR "treatment"[tiab])` |
| Molecular | `("gene"[tiab] OR "protein"[tiab]) AND ("pathway"[tiab] OR "mechanism"[tiab])` |
| Epidemiology | `("condition"[MeSH]) AND (incidence OR prevalence OR "risk factor")` |
| Methods | `("method"[tiab]) AND ("application"[tiab]) AND (validation OR comparison)` |

## Source Selection

### Decision tree
```
Topic is medical/clinical?
├─ Yes → PubMed primary, Google Scholar secondary
└─ No → Topic is CS/physics/math?
    ├─ Yes → arXiv primary, Semantic Scholar secondary
    └─ No → CrossRef primary, Semantic Scholar secondary
```

### Journal scope awareness
- Nature Portfolio journals: use `nature.com` domain filter
- Chinese journals: CNKI/万方 not indexed in PubMed/CrossRef — flag for manual check
- Preprints only: arXiv, bioRxiv, medRxiv — no peer review status available

## Deduplication Logic

See [Dedup Engine](dedup-engine.md) for the unified deduplication strategy shared by Workflows 1, 2, and 5a.

## Result Ranking

### Default: relevance
Use the search engine's default relevance ranking.

### Date-weighted
When user requests "recent" or "latest": sort by publication date descending.

### Citation-weighted
When user cares about impact: sort by citation count descending (available via CrossRef or Semantic Scholar).

### Combined scoring
For systematic reviews: `score = relevance * 0.5 + recency * 0.3 + citations * 0.2`

# Source Tiers & Reliability

Every academic data source is classified by reliability tier to guide automated fallback routing.

## Tier Definitions

| Tier | Label | API Quality | Fallback Behavior |
|------|-------|-------------|-------------------|
| **T1** | API-backed, structured | Official REST/XML API, stable schema | Use first. If fails → next T1 source. |
| **T2** | API-backed, limited | Official API but narrow coverage or low rate limits | Use when T1 exhausted or insufficient. |
| **T3** | Scraped, unstable | Web scraping, no contract on response format | Last resort. Always warn user: "results may be incomplete or stale". |

## Source Classification

### T1 — API-backed, Structured

| Source | API | Rate Limit | Notes |
|--------|-----|------------|-------|
| PubMed | E-utilities (XML/JSON) | 3 req/s (10 with API key) | Biomedical + life sciences, MeSH indexing |
| CrossRef | REST API (JSON) | 50 req/s (no key needed) | Cross-disciplinary, citation counts |
| arXiv | OAI-PMH Atom XML | 1 req/3s | Preprints: physics, math, CS, biology |

### T2 — API-backed, Limited

| Source | API | Rate Limit | Notes |
|--------|-----|------------|-------|
| Semantic Scholar | REST API (JSON) | 1 req/s (100 with API key) | Citation graph, field-of-study filters |
| bioRxiv | API | Limited metadata | Biology preprints only |
| medRxiv | API | Limited metadata | Medical preprints only |

### T3 — Scraped, Unstable

| Source | Method | Risk |
|--------|--------|------|
| Google Scholar | HTML scrape | CAPTCHA blocks, IP bans |
| Web of Science | Institution proxy required | Access varies |
| Scopus | Institution proxy required | Access varies |
| CNKI / 万方 | No programmatic access | Chinese only, manual download |

## Fallback Routing Rules

For every literature search or citation verification:

```
1. SELECT T1 sources matching the query domain
2. SEARCH all selected T1 sources in parallel
3. If (result found AND relevance > threshold) → ACCEPT
4. If T1 exhausted or insufficient → ESCALATE to T2
5. If T1+T2 exhausted → ESCALATE to T3 + WARN USER
6. If all exhausted → return partial results + suggest query refinement
```

### Domain → Tier Mapping

| Domain | T1 | T2 | T3 (if needed) |
|--------|-----|-----|-----------------|
| Medical / clinical | PubMed | Semantic Scholar | Google Scholar |
| Cross-disciplinary | CrossRef | Semantic Scholar | Scopus |
| Preprints / CS / physics | arXiv | bioRxiv / medRxiv | Google Scholar |
| Exhaustive review | PubMed + CrossRef + arXiv | Semantic Scholar + bioRxiv/medRxiv | WoS / Scopus |
| Citation verification | CrossRef (DOI) → PubMed (PMID) | Semantic Scholar | Google Scholar |
| Chinese literature | — | — | CNKI / 万方 (manual) |

# Workflow 1: Multi-Source Literature Search

**Purpose:** Search multiple academic databases in parallel, deduplicate, merge, and rank results.

**Prerequisites:** MCP tools available (PubMed, CrossRef, arXiv, and optionally Semantic Scholar / Google Scholar).

**Uses:** [Dedup Engine](../dedup-engine.md) — deduplication and merge preference logic.

## Procedure

1. **Analyze topic** — identify domain, consult [source routing](../search-strategy.md#source-selection).
2. **Select sources by tier** — follow [Source Tiers](../source-tiers.md). Always try T1 first; escalate to T2 only if T1 insufficient; use T3 as last resort with explicit user warning.
3. **Search in parallel** — call all relevant MCP search tools simultaneously:
   - Biomedical → `pubmed_search_articles`
   - Cross-disciplinary → `search_crossref`
   - Preprints → `search_arxiv` / `search_biorxiv` / `search_medrxiv`
   - Exhaustive → add `search_semantic_scholar` / `search_webofscience` / `search_scopus`
4. **Deduplicate** — apply [Dedup Engine](../dedup-engine.md) to merged result list.
5. **Merge and rank** — sort by relevance, date, or citation count per user preference. See [Result Ranking](../search-strategy.md#result-ranking).
6. **Present results** — unified table with source labels, metadata, and abstract snippets.

## Output Format

```
**Title**: [Paper Title]
**Authors**: [Author list]
**Journal**: [Journal name]
**Year**: [Year]  |  **DOI**: [DOI]  |  **PMID**: [PMID]
**Citations**: [count if available]
**Abstract**: [First 200 characters...]
```

## Error Modes

- **MCP tool unavailable:** report specific failure, continue with remaining tools.
- **No results:** broaden terms per [Query Construction](../search-strategy.md#query-construction), try alternative sources, suggest user refine query.
- **All sources empty:** suggest MeSH strategy (Workflow 3) or manual query refinement.

# Workflow 2: Citation Verification

**Purpose:** Verify references in a document (.docx / .tex / .txt) against databases.

**Uses:**
- [Citation Parser](../citation-parser.md) — extraction strategies per source format.
- [Dedup Engine](../dedup-engine.md) — collapse duplicate candidate matches before classification.

## Procedure

1. **Extract citations** from document using [Citation Parser](../citation-parser.md).
   Prefer T1 sources for primary verification (CrossRef DOI lookup → PubMed PMID confirmation). Use T2 (Semantic Scholar) for cross-checking ambiguous or missing results. See [Source Tiers](../source-tiers.md) for full routing.
2. **Resolve each citation:**
   - DOI → `search_crossref` or `get_paper_by_doi`
   - PMID → `pubmed_fetch_articles`
   - arXiv ID → `search_arxiv`
   - Title + first author → `pubmed_search_articles` or `search_crossref`
3. **Compare** retrieved metadata vs. document metadata (title, journal, year).
4. **Classify** into: `verified` | `mismatch` | `not_found` | `suspicious` | `manual_needed`.
   See [Citation Parser: Classification Labels](../citation-parser.md#classification-labels) for criteria.
5. **Generate report:**
   - Summary: total / verified / mismatched / not_found / suspicious / manual_needed counts.
   - Detail table: each reference with status, DOI/PMID, resolution notes.

## Error Modes

- **Unsupported document format:** report and request .docx, .tex, or .txt.
- **All references manual_needed:** document may lack identifiers; suggest adding DOIs or PMIDs to the manuscript.
- **MCP tools partially unavailable:** flag affected references as `manual_needed`.

# Workflow 3: MeSH Search Strategy

**Purpose:** Build precise PubMed queries from MeSH terms.

## Procedure

1. Use `pubmed_lookup_mesh` to explore terms related to the topic.
2. Show term hierarchy (broader / narrower / related).
3. Construct Boolean query: MeSH terms + keywords.
   See [Query Construction](../search-strategy.md#query-construction) for templates.
4. Optionally spell-check query with `pubmed_spell_check`.
5. Execute via `pubmed_search_articles`.

## Output

Final PubMed query string, result count, and top results.

# Workflow 4: Citation File Management

**Purpose:** Download and convert citation files.

**Uses:** `scripts/format-converter.py` — multi-source downloader (PubMed/CrossRef/arXiv) with .nbib/.ris/.bib output.

## Procedure

1. **Identify papers** — by PMID, DOI, arXiv ID, or search query.
2. **Download** via format-converter:
   ```bash
   # PubMed
   python scripts/format-converter.py --pmid 28344011 --format nbib

   # CrossRef
   python scripts/format-converter.py --doi 10.1038/nature14539 --format ris

   # arXiv
   python scripts/format-converter.py --arxiv 1706.03762 --format bib

   # Batch from file
   python scripts/format-converter.py --input refs.txt --format ris
   ```
3. **Convert format** as needed: `.nbib` (MEDLINE), `.ris` (EndNote/Zotero), `.bib` (BibTeX/LaTeX).
   Format specifications: [RIS and BibTeX Format](../ris-bibtex-format.md).
4. Save to `./references/` directory.
5. Verify output count matches input.

## refs.txt Format

```
PMID:28344011
DOI:10.1038/nature14539
ARXIV:1706.03762
QUERY:TB-Profiler AND Bioinformatics[Journal]
AUTHOR:Dheda TITLE:drug-resistant tuberculosis
# Lines starting with # are comments
```

## Error Modes

- **Script failure (2x):** fall back to manual .ris/.bib generation from MCP-fetched metadata.
- **DOI not found in CrossRef:** suggest verifying DOI spelling, trying PMID instead.
- **arXiv ID not found:** check for version suffix (v1, v2), try without it.

# Workflow 5: Reference Management

**Purpose:** Manage and enrich reference collections.

**Uses:** [Dedup Engine](../dedup-engine.md) — for 5a (related papers overlap).

## 5a. Find Related Papers

1. Fetch source paper metadata via `pubmed_fetch_articles`.
2. Discover related articles via `pubmed_find_related`.
3. Filter by relevance, date, or journal.
4. Deduplicate against source using [Dedup Engine](../dedup-engine.md).
5. Present with context notes.

## 5b. BibTeX Generation

1. DOI → `search_crossref` → format as BibTeX.
2. PMID → `pubmed_fetch_articles` → format as BibTeX.
3. Batch: process multiple IDs via `scripts/format-converter.py`.
4. Clean: deduplicate by citation key, sort, validate required fields.
   See [BibTeX Format](../ris-bibtex-format.md#bibtex-format) for field requirements.

## 5c. ID Conversion

1. Accept DOI, PMID, or PMCID (up to 50).
2. Use `pubmed_convert_ids` for conversion.
3. Fetch metadata for newly resolved IDs via `pubmed_fetch_articles`.

## 5d. Citation Formatting

1. Accept PMIDs.
2. Use `pubmed_format_citations` for APA / MLA / BibTeX / RIS output.

## 5e. Full-Text Access

1. `pubmed_fetch_fulltext` for articles with PMC copies (structured JATS).
2. Fall back to `download_paper` for paywalled articles.
3. Report: structured text / PDF-as-text / metadata-only.

# Workflow 6: Strict Other-Citation Impact Audit

## Contents

- [Definitions](#definitions)
- [Procedure](#procedure)
- [Article Summary Table Mode](#article-summary-table-mode)
- [Output Format](#output-format)
- [Red Lines](#red-lines)


**Purpose:** Audit who cites a target paper, distinguish strict independent
other-citations from self/team citations, build article-level citation metric
tables, identify high-profile independent citers, and extract how those citers
discuss the target paper.

**Use when the user asks:** `严格他引`, `他引判定`, `排除自引`,
`谁引用了我的文章`, `引用我的文章的人有没有大牛`, `院士引用`,
`杰青引用`, `长江学者引用`, `Fellow citation`, `influential citer`,
`citation context`, `文章引用表`, `指定文章引用数`, `严格他引数`,
`整理成表格`, or similar citation-impact questions.

**Uses:**
- [Search Strategy](../search-strategy.md) — construct title/DOI/cited-by queries.
- [Dedup Engine](../dedup-engine.md) — deduplicate citing records across sources.
- [Source Tiers](../source-tiers.md) — report source coverage and confidence limits.

## Definitions

Use conservative labels. Do not upgrade a citation to strict independent status
unless the evidence supports it.

| Label | Meaning |
|---|---|
| `confirmed_strict_other_citation` | The citing paper cites the target paper; no target-paper author overlap; no obvious same lab/team/institution conflict; no discoverable recent coauthorship tie with target authors in the checked sources. |
| `probable_external_citation` | No author overlap is visible, but affiliation, ORCID, coauthor-network, or full metadata is incomplete. |
| `self_or_team_citation` | At least one target-paper author appears on the citing paper, or the citing authors are clearly from the same lab/team/project group. |
| `collaborator_or_affiliation_overlap` | No direct author overlap, but there is a same-institution, same-center, advisor/student, project, or recent coauthorship tie strong enough to make strict independence doubtful. |
| `unknown_metadata` | Metadata is insufficient to classify independence. |

For `严格他引`, the default bar is stricter than a simple non-self-citation:
exclude direct self-citations, same-team citations, and obvious close
collaboration-network citations. If the user wants a looser bibliometric
definition, state the changed rule explicitly.

## Procedure

1. **Identify the target paper.**
   - Resolve title, DOI, PMID/arXiv ID if any, publication year, journal, author
     list, corresponding authors, affiliations, and author identifiers
     (ORCID/Scopus Author ID/Semantic Scholar author ID when available).
   - If the target is ambiguous, list the candidate records and ask for the DOI
     or exact title before auditing.

2. **Retrieve citing papers with source coverage stated.**
   - Prefer citation-graph sources when available: Scopus, Web of Science,
     Semantic Scholar, OpenAlex, CrossRef cited-by, publisher cited-by pages.
   - Use multiple sources when possible, then deduplicate with
     [Dedup Engine](../dedup-engine.md).
   - Report which sources were checked and which were unavailable. Do not imply
     an exhaustive citation count when only partial sources were available.

3. **Classify strict other-citation status.**
   - Compare citing-paper authors against target-paper authors by normalized
     name plus ORCID/author IDs when available.
   - Check affiliations, lab/center names, corresponding-author groups, and
     recent coauthorship ties when source metadata allows.
   - Classify each citing paper using the labels above. Include the specific
     evidence for exclusions, such as shared author, shared lab, shared center,
     or missing metadata.

4. **Extract citation context.**
   - Prefer full text: PubMed Central JATS, ScienceDirect metadata/full text,
     publisher HTML, or legally available PDF text.
   - Locate the target reference in the bibliography, map it to in-text citation
     markers (numeric or author-year), then extract the citing sentence plus one
     adjacent sentence before/after when available.
   - If only abstract/metadata is available, set `citation_context:
     unavailable` and do not infer wording from title similarity.
   - Preserve exact citation-context snippets only when short; otherwise
     paraphrase and identify where the citation appeared.

5. **Identify high-profile independent citers.**
   - Run this step only on `confirmed_strict_other_citation` and
     `probable_external_citation` papers unless the user asks for all citing
     papers.
   - Check identity evidence in this order:
     1. official university, academy, government, funding-agency, or learned
        society pages;
     2. official fellow/member lists, talent-program lists, editorial-board
        pages, or institutional appointment announcements;
     3. stable researcher profiles such as ORCID, Scopus, Semantic Scholar, or
        Google Scholar;
     4. news, personal pages, or third-party biographies only as secondary
        support.
   - Recognized profile signals include academy member / 院士, university
     president or vice president, dean or department chair, national talent
     awards such as 杰青 / 长江学者 / 优青, society fellow status such as IEEE
     Fellow / AAAS Fellow / RSC Fellow / ASCE Fellow, highly cited researcher
     status, major editor roles, or clearly field-leading citation/profile
     evidence.
   - Disambiguate names using affiliation, field, ORCID/author IDs, coauthors,
     and publication topics. If the evidence does not identify the same person,
     mark the profile `unverified`.

6. **Assess how the target paper was cited.**
   - Classify citation function as one or more of:
     `background_support`, `method_used`, `data_or_code_used`,
     `benchmark_or_comparison`, `direct_extension`, `replication_or_validation`,
     `limitation_or_critique`, `review_summary`, or `incidental_mention`.
   - Classify stance as `positive`, `neutral`, `critical`, `mixed`, or
     `not_assessable`.
   - Explain the classification from the citation context. Do not infer praise
     merely from the fact that the paper was cited.

7. **Generate the audit report.**
   - Start with coverage and limitations.
   - Separate high-profile independent citers from ordinary citing papers.
   - Keep every identity claim tied to evidence. Unsupported "大牛" labels are
     not allowed.

## Article Summary Table Mode

Use this compact mode when the user asks to organize one or more specified
papers into a table, especially when they ask for fields such as article title,
publication date, authors, affiliations, citation count, strict other-citation
count, and DOI.

### Required fields

For each specified paper, return a table with these columns unless the user asks
for a different schema:

| Column | Required handling |
|---|---|
| `Article title` | Use the resolved title from DOI/PMID/arXiv/publisher metadata. If multiple records match, ask for confirmation before counting citations. |
| `Publication date` | Prefer full date (`YYYY-MM-DD`); otherwise use year/month or year and mark the precision. |
| `Authors` | List all authors when the list is short; for long author lists, list first 6 + `et al.` and state that the full author list is available. |
| `Author affiliations` | Preserve author-affiliation mapping when metadata provides it. If only institution-level metadata is available, summarize distinct affiliations and mark mapping precision. |
| `Citation count` | Use the best available citation-count source and name it in an evidence note. Do not merge counts from different databases as if they are identical. |
| `Strict other-citation count` | Count only records classified as `confirmed_strict_other_citation`. Optionally report `probable_external_citation` separately in a note, not inside the strict count. |
| `DOI` | Normalize DOI casing/prefix; use `not_found` only after checking title and ID-based lookup routes. |

### Counting rules

1. Resolve the target paper first; do not count citations against an ambiguous
   title match.
2. Retrieve citing records as in the main procedure, deduplicate them, then
   classify strict other-citation status.
3. Set `Citation count` to one source-specific count, preferably Scopus, Web of
   Science, Semantic Scholar, OpenAlex, or publisher/CrossRef in that order when
   available. If multiple counts are available, include them in `Evidence /
   notes` instead of collapsing them.
4. Set `Strict other-citation count` to the number of deduplicated citing papers
   with label `confirmed_strict_other_citation`.
5. If the audit cannot inspect enough metadata to classify strict independence,
   use `not_assessable` for the strict count and state the missing sources.
6. For batches, keep one row per target paper and put source limitations in a
   short notes column or a separate evidence section.

### Table format

```markdown
| Article title | Publication date | Authors | Author affiliations | Citation count | Strict other-citation count | DOI | Evidence / notes |
|---|---|---|---|---:|---:|---|---|
|  |  |  |  |  |  |  |  |
```

For Chinese outputs, use this header:

```markdown
| 文章名 | 发表时间 | 作者名 | 作者机构 | 引用数 | 严格他引数 | DOI | 证据 / 备注 |
|---|---|---|---|---:|---:|---|---|
|  |  |  |  |  |  |  |  |
```

Always include a short source note after the table:

```text
计数来源：
- 引用数来源：
- 严格他引判定来源：
- 未覆盖或无法访问的来源：
```

## Output Format

```text
Strict other-citation impact audit

Target paper
- Title:
- DOI / PMID / arXiv:
- Year / venue:
- Target authors checked:

Coverage
- Citation sources checked:
- Full-text sources checked:
- Identity sources checked:
- Known limitations:

Citation classification summary
- total citing records retrieved:
- deduplicated citing papers:
- confirmed strict other-citations:
- probable external citations:
- self/team citations:
- collaborator or affiliation-overlap citations:
- unknown metadata:

High-profile independent citers
1. [Name]
- Citing paper:
- Strict other-citation status:
- Profile signals:
- Identity evidence:
- Citation context:
- How they used the target paper:
- Stance:
- Confidence:

All citing-paper classifications
| Citing paper | Year | Citer(s) flagged | Strict-status label | Citation context available | Citation function | Notes |
|---|---:|---|---|---|---|---|

Evidence gaps / next checks
- [missing source, paywalled full text, ambiguous author identity, incomplete affiliation metadata]
```

## Red Lines

- Do not call someone an academy member, fellow, dean, president, 杰青, 长江学者,
  or similar without evidence for the same person.
- Do not treat same-name matches as identity matches without affiliation or
  author-ID support.
- Do not treat citation count or h-index alone as proof that the citer is a
  "大牛"; present it as bibliometric context only.
- Do not infer citation context from abstracts, titles, or reference-list
  presence when full text is unavailable.
- Do not hide source coverage limits. If Scopus/WoS/Semantic Scholar/OpenAlex
  are unavailable or incomplete, say so before reporting counts.

#!/usr/bin/env python3
# /// script
# requires-python = ">=3.10"
# ///
"""No-MCP fallback search for nature-academic-search (OpenAlex, stdlib only).

This is the graceful-degradation path for agents/environments that do not mount
the MCP server (plain CLI use, skill auto-discovery, CI). It mirrors what
`nature-citation/scripts/nature_citation.py` already does for CrossRef: a single
self-contained file hitting a public HTTP API with no third-party dependencies.

Source: OpenAlex (https://openalex.org) — free, no API key. OpenAlex indexes
CrossRef, PubMed and arXiv-deposited works, so one endpoint covers journals,
preprints and most of the T1/T2/T3 sources for *discovery*. Pair it with
`format-converter.py` to download/convert the chosen DOIs/PMIDs into
.ris/.bib/.enw citation files — together they make the search→export workflow
runnable without MCP.

Usage:
    # By topic
    python3 academic_search.py "deep potential molecular dynamics" [--limit 10] [--year-from 2020] [--sort cited_by_count|relevance_score|publication_date]
    # By author (resolves the name to OpenAlex author IDs, merging same-name/same-institution entries)
    python3 academic_search.py --author "Wanshui Han"
    # By author + topic within that author's works
    python3 academic_search.py "wave load" --author "Wanshui Han" --sort publication_date
    # By author CONSTRAINED to an institution (disambiguates common/romanised names)
    python3 academic_search.py --author "Shenghua Zhou" --affiliation "Hong Kong"
    # List every same-name author cluster, then pick the right institution / ID
    python3 academic_search.py --author "Shenghua Zhou" --list-authors
    # By ORCID (unambiguous, best for common Chinese names)
    python3 academic_search.py --orcid 0000-0002-1825-0097
    # By exact author ID (skip name resolution)
    python3 academic_search.py --author-id A5055881494

Be polite to the OpenAlex pool: pass --mailto [email protected] or set the
OPENALEX_MAILTO / CROSSREF_MAILTO environment variable.

Output: JSON list of papers with title, DOI, authors, year, citation count, abstract.
Per-source failures (HTTP 429 rate-limit, timeouts, network errors) are reported
on stderr and exit non-zero, so a caller can treat this source independently and
fall back to another tool.
"""

import argparse
import json
import os
import sys
import urllib.request
import urllib.parse
import urllib.error


OPENALEX_API = "https://api.openalex.org/works"
AUTHORS_API = "https://api.openalex.org/authors"
# Polite-pool contact; overridden by --mailto or OPENALEX_MAILTO / CROSSREF_MAILTO.
MAILTO = "[email protected]"

# When re-ranking a query's results by citations/date, drop candidates whose
# relevance_score is below this fraction of the top hit's score. OpenAlex's
# relevance_score carries a citation boost, so a handful of weakly-matching but
# ultra-cited papers (e.g. a 3000-cite medical paper for a civil-engineering query)
# sneak into the pool; without this floor, re-ranking by citations floats them to
# the top. Empirically off-topic papers score ~0.2x the top hit while on-topic ones
# stay above ~0.3x, so 0.3 cleanly separates them.
RELEVANCE_FLOOR = 0.3


def _institution(author: dict) -> str:
    insts = author.get("last_known_institutions") or []
    if isinstance(insts, list) and insts:
        return insts[0].get("display_name", "") or ""
    # Fall back to the most recent listed affiliation if no last-known institution.
    for aff in (author.get("affiliations") or []):
        n = (aff.get("institution") or {}).get("display_name")
        if n:
            return n
    return ""


def _aff_match(author: dict, affiliation: str) -> bool:
    """True if `affiliation` matches the author's PRIMARY (last-known) institution.

    Matching only the last-known institution-rather than every historical
    affiliation-keeps a high-volume namesake with a tenuous past link to the
    target institution from hijacking the result (e.g. a 273-paper radar
    "Shenghua Zhou" who once brushed against "Hong Kong" must not outrank the
    real civil-engineering one whose last-known institution *is* Hong Kong).
    """
    aff = affiliation.lower()
    names = [inst.get("display_name", "") for inst in (author.get("last_known_institutions") or [])]
    if not names:
        names = [_institution(author)]  # fall back to most recent listed affiliation
    return any(aff in (n or "").lower() for n in names)


def _cluster_summary(results: list[dict]) -> list[dict]:
    """Collapse author records into (name, institution) clusters, most-prolific first.

    Lets callers see the *distinct* people behind a colliding name (e.g. a
    medical and a civil-engineering "Shenghua Zhou") instead of silently
    inheriting whichever cluster happens to be largest.
    """
    clusters: dict[tuple, dict] = {}
    for a in results:
        key = ((a.get("display_name") or "").lower(), _institution(a))
        c = clusters.setdefault(key, {
            "name": a.get("display_name", ""),
            "institution": _institution(a),
            "works_count": 0,
            "ids": [],
        })
        c["works_count"] += a.get("works_count") or 0
        c["ids"].append((a.get("id") or "").rsplit("/", 1)[-1])
    return sorted(clusters.values(), key=lambda c: -c["works_count"])


def fetch_author_candidates(name: str, per_page: int = 50) -> list[dict]:
    params = {"search": name, "per_page": per_page, "mailto": MAILTO}
    url = f"{AUTHORS_API}?{urllib.parse.urlencode(params)}"
    req = urllib.request.Request(url, headers={"Accept": "application/json"})
    with urllib.request.urlopen(req, timeout=30) as resp:
        data = json.loads(resp.read().decode())
    return data.get("results", [])


def resolve_author(name: str, affiliation: str | None = None) -> dict | None:
    """Resolve an author name to OpenAlex author ID(s).

    OpenAlex often splits one real person into several records, and common
    (e.g. romanised Chinese) names collide across unrelated people. We therefore:
      1. optionally keep only candidates whose affiliation matches `affiliation`
         (case-insensitive substring over the last-known institution); then
      2. pick the most prolific institution-bearing candidate as the anchor; then
      3. merge namesakes that share the anchor's name and institution.
    The returned dict always carries a `candidates` cluster summary so callers
    can detect/repair a wrong pick. When `affiliation` matches nothing, returns
    a dict with empty `ids` and `no_affiliation_match` set (not None).
    """
    results = fetch_author_candidates(name)
    if not results:
        return None

    clusters = _cluster_summary(results)
    pool = results
    if affiliation:
        pool = [a for a in results if _aff_match(a, affiliation)]
        if not pool:
            return {"ids": [], "display_name": name, "institution": "",
                    "works_count": 0, "candidates": clusters,
                    "no_affiliation_match": affiliation}

    # Prefer candidates that have a known institution, then the most prolific.
    pool = sorted(pool, key=lambda a: (_institution(a) == "", -(a.get("works_count") or 0)))
    best = pool[0]
    best_name = (best.get("display_name") or "").lower()
    best_inst = _institution(best)

    ids, works = [], 0
    for a in results:
        if (a.get("display_name") or "").lower() != best_name:
            continue
        # Only merge namesakes that share the institution; if the anchor has
        # no institution we can't disambiguate, so keep that single record.
        if best_inst:
            if _institution(a) != best_inst:
                continue
        elif a is not best:
            continue
        ids.append((a.get("id") or "").rsplit("/", 1)[-1])
        works += a.get("works_count") or 0

    return {
        "ids": ids,
        "display_name": best.get("display_name", ""),
        "institution": best_inst,
        "works_count": works,
        "candidates": clusters,
    }


def resolve_by_orcid(orcid: str) -> dict | None:
    """Resolve an ORCID (bare 0000-... or full URL) to its OpenAlex author ID(s)."""
    oid = orcid.strip().rsplit("/", 1)[-1]
    # 0000-0000-0000-0000 is a checksum-valid but unassigned placeholder some
    # records carry; reject it so it can't resolve to whoever holds that junk tag.
    if set(oid) <= {"0", "-"}:
        return None
    params = {"filter": f"orcid:{oid}", "mailto": MAILTO}
    url = f"{AUTHORS_API}?{urllib.parse.urlencode(params)}"
    req = urllib.request.Request(url, headers={"Accept": "application/json"})
    with urllib.request.urlopen(req, timeout=30) as resp:
        data = json.loads(resp.read().decode())
    results = data.get("results", [])
    # Defensive: only keep records that actually carry the requested ORCID, so a
    # server-side filter miss can't resolve to an unrelated author. OpenAlex
    # stores orcid as a full URL, e.g. https://orcid.org/0000-....
    results = [a for a in results if (a.get("orcid") or "").rstrip("/").endswith(oid)]
    if not results:
        return None
    best = results[0]
    return {
        "ids": [(a.get("id") or "").rsplit("/", 1)[-1] for a in results],
        "display_name": best.get("display_name", ""),
        "institution": _institution(best),
        "works_count": sum(a.get("works_count") or 0 for a in results),
        "candidates": _cluster_summary(results),
    }


def search(query: str | None = None, limit: int = 10, year_from: int | None = None,
           sort: str = "relevance_score", author_id: str | None = None) -> list[dict]:
    # A text query is ranked by relevance. Asking OpenAlex to sort that query by
    # cited_by_count / publication_date *server-side* discards relevance entirely
    # and surfaces off-topic mega-cited (or merely newest) papers that only loosely
    # match. So when a query and a non-relevance sort are combined, fetch a larger
    # relevance-ranked candidate pool and re-sort it locally (see end of function),
    # keeping only the topically relevant top results. Without a query (e.g. author
    # browse) there is no relevance to preserve, so sort server-side as before.
    rerank = bool(query) and sort != "relevance_score"
    per_page = min(max(limit * 5, limit), 200) if rerank else min(limit, 50)

    params = {
        "per_page": per_page,
        "mailto": MAILTO,
    }
    if query:
        params["search"] = query
    if not rerank:
        if sort != "relevance_score":
            params["sort"] = sort + ":desc"
        elif not query:
            # Author-only browse: relevance is meaningless, default to most-cited.
            params["sort"] = "cited_by_count:desc"

    filters = []
    if year_from:
        filters.append(f"from_publication_date:{year_from}-01-01")
    if author_id:
        # Accept a bare ID, a full URL, or an OR-list; normalise each segment.
        ids = "|".join(p.rsplit("/", 1)[-1] for p in author_id.split("|"))
        filters.append(f"author.id:{ids}")
    if filters:
        params["filter"] = ",".join(filters)

    url = f"{OPENALEX_API}?{urllib.parse.urlencode(params)}"
    req = urllib.request.Request(url, headers={"Accept": "application/json"})
    with urllib.request.urlopen(req, timeout=30) as resp:
        data = json.loads(resp.read().decode())

    results = []
    for work in data.get("results", []):
        authors = [
            a.get("author", {}).get("display_name", "")
            for a in work.get("authorships", [])
        ]
        doi = work.get("doi", "")
        if doi and doi.startswith("https://doi.org/"):
            doi = doi[len("https://doi.org/"):]

        abstract_index = work.get("abstract_inverted_index")
        abstract = ""
        if abstract_index:
            # Reconstruct abstract from inverted index
            word_positions = []
            for word, positions in abstract_index.items():
                for pos in positions:
                    word_positions.append((pos, word))
            word_positions.sort()
            abstract = " ".join(w for _, w in word_positions)

        results.append({
            "title": work.get("title", ""),
            "doi": doi,
            "authors": authors[:5],  # first 5 authors
            "year": work.get("publication_year"),
            "publication_date": work.get("publication_date", ""),
            "cited_by_count": work.get("cited_by_count", 0),
            # OpenAlex returns source=null for repository deposits / some preprints,
            # so guard with `or {}` instead of relying on .get's default.
            "journal": ((work.get("primary_location") or {}).get("source") or {}).get("display_name", ""),
            "abstract": abstract[:500] if abstract else "",
            "openalex_id": work.get("id", ""),
            # Present only for query searches (server orders by relevance then).
            "relevance_score": work.get("relevance_score"),
        })

    if rerank:
        # Drop weak matches before re-ranking, so an off-topic ultra-cited paper
        # can't float to the top (see RELEVANCE_FLOOR).
        scores = [r["relevance_score"] for r in results if r.get("relevance_score")]
        if scores:
            cutoff = RELEVANCE_FLOOR * max(scores)
            results = [r for r in results if (r.get("relevance_score") or 0) >= cutoff]
        sort_keys = {
            "cited_by_count": lambda r: r.get("cited_by_count") or 0,
            "publication_date": lambda r: r.get("publication_date") or "",
        }
        results.sort(key=sort_keys[sort], reverse=True)

    # We may have over-fetched a candidate pool; return only the requested number.
    return results[:limit]


def main():
    parser = argparse.ArgumentParser(description="No-MCP literature search via OpenAlex")
    parser.add_argument("query", nargs="?", default=None,
                        help="Search query (optional when --author/--author-id/--orcid is given)")
    parser.add_argument("--author", default=None,
                        help="Filter by author name (resolved to OpenAlex author IDs)")
    parser.add_argument("--affiliation", default=None,
                        help="Constrain --author to an institution (case-insensitive substring); "
                             "disambiguates colliding names")
    parser.add_argument("--orcid", default=None,
                        help="Resolve author by ORCID (unambiguous); takes precedence over --author")
    parser.add_argument("--list-authors", action="store_true",
                        help="List same-name author clusters for --author and exit (for disambiguation)")
    parser.add_argument("--author-id", default=None,
                        help="Filter by exact OpenAlex author ID(s), e.g. A5055881494 (OR-join with '|')")
    parser.add_argument("--limit", type=int, default=10, help="Number of results (max 50)")
    parser.add_argument("--year-from", type=int, default=None, help="Filter papers from this year")
    parser.add_argument("--sort", default="relevance_score",
                        choices=["relevance_score", "cited_by_count", "publication_date"],
                        help="Sort order. With a query, cited_by_count/publication_date "
                             "re-rank within the relevance-matched pool (not the whole DB)")
    parser.add_argument("--mailto", default=None,
                        help="Contact email for the OpenAlex polite pool "
                             "(falls back to OPENALEX_MAILTO / CROSSREF_MAILTO env)")
    parser.add_argument("--compact", action="store_true", help="Compact output (one line per paper)")
    args = parser.parse_args()

    # Resolve the polite-pool contact: --mailto > OPENALEX_MAILTO > CROSSREF_MAILTO > default.
    global MAILTO
    MAILTO = (args.mailto or os.environ.get("OPENALEX_MAILTO")
              or os.environ.get("CROSSREF_MAILTO") or MAILTO)

    if (not args.query and not args.author and not args.author_id
            and not args.orcid and not args.list_authors):
        parser.error("provide a search query, or use --author / --author-id / --orcid")

    def _fmt_cluster(c: dict) -> str:
        return f"  {(c['institution'] or 'unknown institution'):<42} | {c['works_count']:>5} works | {'|'.join(c['ids'])}"

    # --list-authors: dump the distinct same-name clusters and exit.
    if args.list_authors:
        if not args.author:
            parser.error("--list-authors requires --author")
        try:
            cands = _cluster_summary(fetch_author_candidates(args.author))
        except (urllib.error.HTTPError, urllib.error.URLError, TimeoutError, OSError, json.JSONDecodeError) as e:
            print(f"ERROR: author lookup failed: {e}", file=sys.stderr)
            return 1
        if not cands:
            print(f"ERROR: no author found in OpenAlex: {args.author}", file=sys.stderr)
            return 1
        print(f"Same-name author clusters for \"{args.author}\" (by works count, desc):", file=sys.stderr)
        for c in cands[:15]:
            print(_fmt_cluster(c))
        print("-> use --affiliation \"keyword\" or --author-id <ID> to pin the right person", file=sys.stderr)
        return 0

    author_id = args.author_id
    matched = None
    if not author_id and (args.orcid or args.author):
        try:
            matched = (resolve_by_orcid(args.orcid) if args.orcid
                       else resolve_author(args.author, args.affiliation))
        except urllib.error.HTTPError as e:
            print(f"ERROR: author lookup returned HTTP {e.code}: {e.reason}", file=sys.stderr)
            return 1
        except (urllib.error.URLError, TimeoutError, OSError, json.JSONDecodeError) as e:
            print(f"ERROR: author lookup failed: {e}", file=sys.stderr)
            return 1
        if not matched or not matched.get("ids"):
            if matched and matched.get("no_affiliation_match"):
                print(f"ERROR: author \"{args.author}\" has no record at institution "
                      f"\"{matched['no_affiliation_match']}\". Options (by works count):", file=sys.stderr)
                for c in (matched.get("candidates") or [])[:10]:
                    print(_fmt_cluster(c), file=sys.stderr)
            elif args.orcid:
                print(f"ERROR: no author found in OpenAlex for ORCID: {args.orcid}", file=sys.stderr)
            else:
                print(f"ERROR: no author found in OpenAlex: {args.author}", file=sys.stderr)
            return 1
        author_id = "|".join(matched["ids"])
        inst = matched["institution"] or "unknown institution"
        print(f"Matched author: {matched['display_name']} | {inst} | "
              f"merged {len(matched['ids'])} author record(s), ~{matched['works_count']} works | "
              f"{author_id}", file=sys.stderr)
        # Warn on silent collisions: same name, other institutions present, and the
        # caller didn't pin it down with --affiliation/--orcid.
        if args.author and not args.affiliation and not args.orcid:
            others = [c for c in (matched.get("candidates") or [])
                      if c["institution"] and c["institution"] != matched["institution"]]
            if others:
                tops = "; ".join(f"{c['institution']} ({c['works_count']} works)" for c in others[:3])
                print(f"WARNING: same-name authors at other institutions exist; "
                      f"if this is the wrong one add --affiliation or --orcid: {tops}",
                      file=sys.stderr)

    if args.query and args.sort != "relevance_score":
        print(f"INFO: recalled a relevance-ranked pool first, then re-ranked by {args.sort} "
              f"and kept the top {args.limit} (avoids pulling in off-topic high-cited/newest papers)",
              file=sys.stderr)

    try:
        results = search(args.query, args.limit, args.year_from, args.sort, author_id)
    except urllib.error.HTTPError as e:
        hint = " (429 = rate-limited; retry later or lower --limit)" if e.code == 429 else ""
        print(f"ERROR: OpenAlex returned HTTP {e.code}: {e.reason}{hint}", file=sys.stderr)
        return 1
    except (urllib.error.URLError, TimeoutError, OSError) as e:
        print(f"ERROR: network request failed (check connectivity): {e}", file=sys.stderr)
        return 1
    except json.JSONDecodeError:
        print("ERROR: OpenAlex returned a non-JSON response (service may be down; retry)", file=sys.stderr)
        return 1

    if args.compact:
        for r in results:
            authors_str = ", ".join(r["authors"][:3])
            if len(r["authors"]) > 3:
                authors_str += " et al."
            print(f"[{r['year']}] {r['title']} | {authors_str} | {r['journal']} | DOI:{r['doi']} | Cited:{r['cited_by_count']}")
    else:
        print(json.dumps(results, ensure_ascii=False, indent=2))
    return 0


if __name__ == "__main__":
    sys.exit(main())

# -*- coding: utf-8 -*-
"""
Format converters for academic citations.
Supports: MEDLINE/.nbib -> RIS/BibTeX/ENW, CrossRef JSON -> RIS/BibTeX/ENW, arXiv XML -> RIS/BibTeX/ENW.

Each converter accepts parsed data and returns a formatted string.
"""

import re


def ris_escape(text):
    """Strip HTML tags and normalize whitespace for safe RIS field output."""
    text = re.sub(r"<[^>]+>", " ", text or "")
    text = re.sub(r"\s+", " ", text)
    return text.strip()


def parse_medline_fields(nbib_text):
    """Parse MEDLINE text into a dict of tag -> [values]."""
    fields = {}
    current_tag = None
    current_value = []

    for line in nbib_text.split("\n"):
        if len(line) >= 6 and line[0:4] != "    " and line[4:6] == "- ":
            if current_tag:
                fields.setdefault(current_tag, []).append(" ".join(current_value))
            current_tag = line[0:4].strip()
            current_value = [line[6:].strip()]
        elif line.startswith("      ") and current_tag:
            current_value.append(line[6:].strip())
        elif not line.strip():
            if current_tag:
                fields.setdefault(current_tag, []).append(" ".join(current_value))
            current_tag = None
            current_value = []

    if current_tag:
        fields.setdefault(current_tag, []).append(" ".join(current_value))

    return fields


def _get_first(fields, tag):
    vals = fields.get(tag, [])
    return vals[0] if vals else ""


def _extract_doi(fields):
    # MEDLINE often emits the PII variant of LID/AID before the DOI variant
    # (e.g. ScienceDirect/Cell Press records); scan every value, not just the
    # first one, so the DOI is not silently dropped.
    for tag in ("LID", "AID"):
        for val in fields.get(tag, []):
            if val and "[doi]" in val:
                return val.replace(" [doi]", "")
    return ""


def _extract_year(fields):
    dp = _get_first(fields, "DP")
    return dp[:4] if dp else ""


# ── MEDLINE -> RIS ──────────────────────────────────────────────

def medline_to_ris(fields):
    lines = ["TY  - JOUR"]

    for au in fields.get("AU", []):
        lines.append(f"AU  - {ris_escape(au)}")

    ti = _get_first(fields, "TI")
    if ti:
        lines.append(f"TI  - {ris_escape(ti)}")

    jt = _get_first(fields, "JT")
    ta = _get_first(fields, "TA")
    if jt:
        lines.append(f"JO  - {ris_escape(jt)}")
        lines.append(f"T2  - {ris_escape(jt)}")
    if ta:
        lines.append(f"JA  - {ris_escape(ta)}")

    year = _extract_year(fields)
    if year:
        lines.append(f"PY  - {year}")

    vi = _get_first(fields, "VI")
    ip = _get_first(fields, "IP")
    if vi:
        lines.append(f"VL  - {vi}")
    if ip:
        lines.append(f"IS  - {ip}")

    pg = _get_first(fields, "PG")
    if pg and "-" in pg:
        sp, ep = pg.split("-", 1)
        lines.append(f"SP  - {sp.strip()}")
        lines.append(f"EP  - {ep.strip()}")
    elif pg:
        lines.append(f"SP  - {pg}")

    doi = _extract_doi(fields)
    if doi:
        lines.append(f"DO  - {doi}")
        lines.append(f"UR  - https://doi.org/{doi}")

    ab = _get_first(fields, "AB")
    if ab:
        lines.append(f"N2  - {ris_escape(ab)[:500]}")

    for mh in fields.get("MH", []):
        lines.append(f"KW  - {ris_escape(mh)}")

    pmid = _get_first(fields, "PMID")
    if pmid:
        lines.append(f"AN  - PMID:{pmid}")

    lines.append("DB  - PubMed")
    lines.append("ER  - ")
    return "\n".join(lines) + "\n"


# ── MEDLINE -> BibTeX ────────────────────────────────────────────

def medline_to_bib(fields):
    pmid = _get_first(fields, "PMID")
    citation_key = f"pmid{pmid}" if pmid else "unknown"

    lines = [f"@article{{{citation_key},"]

    aus = fields.get("AU", [])
    if aus:
        lines.append(f"  author  = {{ {' and '.join(aus)} }},")

    ti = _get_first(fields, "TI")
    if ti:
        lines.append(f"  title   = {{{{{ti}}}}},")

    jt = _get_first(fields, "JT")
    if jt:
        lines.append(f"  journal = {{{jt}}},")

    year = _extract_year(fields)
    if year:
        lines.append(f"  year    = {{{year}}},")

    vi = _get_first(fields, "VI")
    if vi:
        lines.append(f"  volume  = {{{vi}}},")

    ip = _get_first(fields, "IP")
    if ip:
        lines.append(f"  number  = {{{ip}}},")

    pg = _get_first(fields, "PG")
    if pg:
        lines.append(f"  pages   = {{{pg.replace('-', '--')}}},")

    doi = _extract_doi(fields)
    if doi:
        lines.append(f"  doi     = {{{doi}}},")

    ab = _get_first(fields, "AB")
    if ab:
        lines.append(f"  abstract = {{{ab}}},")

    if pmid:
        lines.append(f"  pmid    = {{{pmid}}},")

    lines.append("}")
    return "\n".join(lines) + "\n"


# ── CrossRef JSON -> RIS ─────────────────────────────────────────

def crossref_to_ris(data):
    """Convert CrossRef API JSON response to RIS format."""
    msg = data.get("message", data)
    lines = ["TY  - JOUR"]

    for author in msg.get("author", []):
        family = author.get("family", "")
        given = author.get("given", "")
        if family:
            lines.append(f"AU  - {ris_escape(family)}, {ris_escape(given)}")

    title = msg.get("title", [])
    if title:
        lines.append(f"TI  - {ris_escape(title[0])}")

    container = msg.get("container-title", [])
    short_container = msg.get("short-container-title", [])
    if container:
        lines.append(f"JO  - {ris_escape(container[0])}")
        lines.append(f"T2  - {ris_escape(container[0])}")
    if short_container:
        lines.append(f"JA  - {ris_escape(short_container[0])}")

    issued = msg.get("issued", {})
    date_parts = issued.get("date-parts", [[None]])[0]
    year = str(date_parts[0]) if date_parts and date_parts[0] else ""
    if year:
        lines.append(f"PY  - {year}")

    volume = msg.get("volume", "")
    issue = msg.get("issue", "")
    if volume:
        lines.append(f"VL  - {volume}")
    if issue:
        lines.append(f"IS  - {issue}")

    page = msg.get("page", "")
    if page and "-" in page:
        sp, ep = page.split("-", 1)
        lines.append(f"SP  - {sp.strip()}")
        lines.append(f"EP  - {ep.strip()}")
    elif page:
        lines.append(f"SP  - {page}")

    doi = msg.get("DOI", "")
    if doi:
        lines.append(f"DO  - {doi}")
        lines.append(f"UR  - https://doi.org/{doi}")

    abstract = msg.get("abstract", "")
    if abstract:
        abstract = re.sub(r"<[^>]+>", "", abstract)
        lines.append(f"N2  - {ris_escape(abstract)[:500]}")

    lines.append("DB  - CrossRef")
    lines.append("ER  - ")
    return "\n".join(lines) + "\n"


# ── CrossRef JSON -> BibTeX ──────────────────────────────────────

def crossref_to_bib(data):
    """Convert CrossRef API JSON response to BibTeX format."""
    msg = data.get("message", data)

    first_author = msg.get("author", [{}])[0].get("family", "unknown") if msg.get("author") else "unknown"
    issued = msg.get("issued", {})
    date_parts = issued.get("date-parts", [[None]])[0]
    year = str(date_parts[0]) if date_parts and date_parts[0] else ""
    citation_key = f"{first_author.lower()}{year}"

    lines = [f"@article{{{citation_key},"]

    authors = []
    for author in msg.get("author", []):
        family = author.get("family", "")
        given = author.get("given", "")
        if family:
            authors.append(f"{family}, {given}")
    if authors:
        lines.append(f"  author  = {{ {' and '.join(authors)} }},")

    title = msg.get("title", [])
    if title:
        lines.append(f"  title   = {{{{{title[0]}}}}},")

    container = msg.get("container-title", [])
    if container:
        lines.append(f"  journal = {{{container[0]}}},")

    if year:
        lines.append(f"  year    = {{{year}}},")

    volume = msg.get("volume", "")
    if volume:
        lines.append(f"  volume  = {{{volume}}},")

    issue = msg.get("issue", "")
    if issue:
        lines.append(f"  number  = {{{issue}}},")

    page = msg.get("page", "")
    if page:
        lines.append(f"  pages   = {{{page.replace('-', '--')}}},")

    doi = msg.get("DOI", "")
    if doi:
        lines.append(f"  doi     = {{{doi}}},")

    abstract = msg.get("abstract", "")
    if abstract:
        abstract = re.sub(r"<[^>]+>", "", abstract)
        lines.append(f"  abstract = {{{abstract}}},")

    lines.append("}")
    return "\n".join(lines) + "\n"


# ── arXiv XML -> RIS ─────────────────────────────────────────────

def arxiv_to_ris(root):
    """Convert arXiv API Atom XML response (xml.etree root) to RIS format."""
    ns = {
        "atom": "http://www.w3.org/2005/Atom",
        "arxiv": "http://arxiv.org/schemas/atom",
    }

    entry = root.find("atom:entry", ns)
    if entry is None:
        return ""

    lines = ["TY  - JOUR"]

    for author in entry.findall("atom:author", ns):
        name = author.find("atom:name", ns)
        if name is not None and name.text:
            parts = name.text.rsplit(" ", 1)
            if len(parts) == 2:
                lines.append(f"AU  - {ris_escape(parts[1])}, {ris_escape(parts[0])}")
            else:
                lines.append(f"AU  - {ris_escape(name.text)}")

    title_el = entry.find("atom:title", ns)
    title = title_el.text.strip() if title_el is not None and title_el.text else ""
    if title:
        lines.append(f"TI  - {ris_escape(title)}")

    lines.append("JO  - arXiv preprint")
    lines.append("T2  - arXiv preprint")

    published = entry.find("atom:published", ns)
    year = published.text[:4] if published is not None and published.text else ""
    if year:
        lines.append(f"PY  - {year}")

    arxiv_id_el = entry.find("atom:id", ns)
    arxiv_id = arxiv_id_el.text.strip() if arxiv_id_el is not None and arxiv_id_el.text else ""
    if "/abs/" in arxiv_id:
        arxiv_id = arxiv_id.split("/abs/")[-1]
    if arxiv_id:
        lines.append(f"DO  - {arxiv_id}")
        lines.append(f"UR  - https://arxiv.org/abs/{arxiv_id}")

    summary_el = entry.find("atom:summary", ns)
    abstract = summary_el.text.strip() if summary_el is not None and summary_el.text else ""
    if abstract:
        lines.append(f"N2  - {ris_escape(abstract)[:500]}")

    lines.append("DB  - arXiv")
    lines.append("ER  - ")
    return "\n".join(lines) + "\n"


# ── arXiv XML -> BibTeX ──────────────────────────────────────────

def arxiv_to_bib(root):
    """Convert arXiv API Atom XML response to BibTeX format."""
    ns = {
        "atom": "http://www.w3.org/2005/Atom",
        "arxiv": "http://arxiv.org/schemas/atom",
    }

    entry = root.find("atom:entry", ns)
    if entry is None:
        return ""

    arxiv_id_el = entry.find("atom:id", ns)
    arxiv_id = arxiv_id_el.text.strip() if arxiv_id_el is not None and arxiv_id_el.text else ""
    if "/abs/" in arxiv_id:
        arxiv_id = arxiv_id.split("/abs/")[-1]
    citation_key = arxiv_id.replace(".", "").replace("/", "") if arxiv_id else "unknown"

    lines = [f"@article{{{citation_key},"]

    authors = []
    for author in entry.findall("atom:author", ns):
        name = author.find("atom:name", ns)
        if name is not None and name.text:
            parts = name.text.rsplit(" ", 1)
            if len(parts) == 2:
                authors.append(f"{parts[1]}, {parts[0]}")
            else:
                authors.append(name.text)
    if authors:
        lines.append(f"  author  = {{ {' and '.join(authors)} }},")

    title_el = entry.find("atom:title", ns)
    title = title_el.text.strip() if title_el is not None and title_el.text else ""
    if title:
        lines.append(f"  title   = {{{{{title}}}}},")

    lines.append("  journal = {arXiv preprint},")

    published = entry.find("atom:published", ns)
    year = published.text[:4] if published is not None and published.text else ""
    if year:
        lines.append(f"  year    = {{{year}}},")

    if arxiv_id:
        lines.append(f"  doi     = {{{arxiv_id}}},")
        lines.append(f"  url     = {{https://arxiv.org/abs/{arxiv_id}}},")

    summary_el = entry.find("atom:summary", ns)
    abstract = summary_el.text.strip() if summary_el is not None and summary_el.text else ""
    if abstract:
        lines.append(f"  abstract = {{{abstract}}},")

    lines.append("}")
    return "\n".join(lines) + "\n"


# ── MEDLINE -> ENW ───────────────────────────────────────────────

def medline_to_enw(fields):
    lines = ["%0 Journal Article"]

    ti = _get_first(fields, "TI")
    if ti:
        lines.append(f"%T {ris_escape(ti)}")

    for au in fields.get("AU", []):
        lines.append(f"%A {ris_escape(au)}")

    jt = _get_first(fields, "JT")
    if jt:
        lines.append(f"%J {ris_escape(jt)}")

    vi = _get_first(fields, "VI")
    if vi:
        lines.append(f"%V {vi}")

    ip = _get_first(fields, "IP")
    if ip:
        lines.append(f"%N {ip}")

    pg = _get_first(fields, "PG")
    if pg:
        lines.append(f"%P {pg}")

    year = _extract_year(fields)
    if year:
        lines.append(f"%D {year}")

    doi = _extract_doi(fields)
    if doi:
        lines.append(f"%R {doi}")
        lines.append(f"%U https://doi.org/{doi}")

    ab = _get_first(fields, "AB")
    if ab:
        lines.append(f"%X {ris_escape(ab)[:500]}")

    return "\n".join(lines) + "\n"


# ── CrossRef JSON -> ENW ──────────────────────────────────────────

def crossref_to_enw(data):
    msg = data.get("message", data)
    lines = ["%0 Journal Article"]

    title = msg.get("title", [])
    if title:
        lines.append(f"%T {ris_escape(title[0])}")

    for author in msg.get("author", []):
        family = author.get("family", "")
        given = author.get("given", "")
        if family:
            lines.append(f"%A {ris_escape(family)}, {ris_escape(given)}")

    container = msg.get("container-title", [])
    if container:
        lines.append(f"%J {ris_escape(container[0])}")

    volume = msg.get("volume", "")
    if volume:
        lines.append(f"%V {volume}")

    issue = msg.get("issue", "")
    if issue:
        lines.append(f"%N {issue}")

    page = msg.get("page", "")
    if page:
        lines.append(f"%P {page}")

    issued = msg.get("issued", {})
    date_parts = issued.get("date-parts", [[None]])[0]
    year = str(date_parts[0]) if date_parts and date_parts[0] else ""
    if year:
        lines.append(f"%D {year}")

    doi = msg.get("DOI", "")
    if doi:
        lines.append(f"%R {doi}")
        lines.append(f"%U https://doi.org/{doi}")

    abstract = msg.get("abstract", "")
    if abstract:
        abstract = re.sub(r"<[^>]+>", "", abstract)
        lines.append(f"%X {ris_escape(abstract)[:500]}")

    return "\n".join(lines) + "\n"


# ── arXiv XML -> ENW ──────────────────────────────────────────────

def arxiv_to_enw(root):
    ns = {
        "atom": "http://www.w3.org/2005/Atom",
        "arxiv": "http://arxiv.org/schemas/atom",
    }

    entry = root.find("atom:entry", ns)
    if entry is None:
        return ""

    lines = ["%0 Journal Article"]

    title_el = entry.find("atom:title", ns)
    title = title_el.text.strip() if title_el is not None and title_el.text else ""
    if title:
        lines.append(f"%T {ris_escape(title)}")

    for author in entry.findall("atom:author", ns):
        name = author.find("atom:name", ns)
        if name is not None and name.text:
            parts = name.text.rsplit(" ", 1)
            if len(parts) == 2:
                lines.append(f"%A {ris_escape(parts[1])}, {ris_escape(parts[0])}")
            else:
                lines.append(f"%A {ris_escape(name.text)}")

    lines.append("%J arXiv preprint")

    published = entry.find("atom:published", ns)
    year = published.text[:4] if published is not None and published.text else ""
    if year:
        lines.append(f"%D {year}")

    arxiv_id_el = entry.find("atom:id", ns)
    arxiv_id = arxiv_id_el.text.strip() if arxiv_id_el is not None and arxiv_id_el.text else ""
    if "/abs/" in arxiv_id:
        arxiv_id = arxiv_id.split("/abs/")[-1]
    if arxiv_id:
        lines.append(f"%R {arxiv_id}")
        lines.append(f"%U https://arxiv.org/abs/{arxiv_id}")

    summary_el = entry.find("atom:summary", ns)
    abstract = summary_el.text.strip() if summary_el is not None and summary_el.text else ""
    if abstract:
        lines.append(f"%X {ris_escape(abstract)[:500]}")

    return "\n".join(lines) + "\n"


# ── Format dispatch ─────────────────────────────────────────────

def convert_from_medline(nbib_text, fmt):
    """Convert MEDLINE/.nbib text to RIS, BibTeX, or ENW."""
    fields = parse_medline_fields(nbib_text)
    if fmt == "ris":
        return medline_to_ris(fields)
    elif fmt == "bib":
        return medline_to_bib(fields)
    elif fmt == "enw":
        return medline_to_enw(fields)
    return nbib_text


def convert_from_crossref(json_data, fmt):
    """Convert CrossRef JSON to RIS, BibTeX, or ENW."""
    if fmt == "ris":
        return crossref_to_ris(json_data)
    elif fmt == "bib":
        return crossref_to_bib(json_data)
    elif fmt == "enw":
        return crossref_to_enw(json_data)
    raise ValueError(f"Unsupported CrossRef format: {fmt}")


def convert_from_arxiv(xml_root, fmt):
    """Convert arXiv Atom XML to RIS, BibTeX, or ENW."""
    if fmt == "ris":
        return arxiv_to_ris(xml_root)
    elif fmt == "bib":
        return arxiv_to_bib(xml_root)
    elif fmt == "enw":
        return arxiv_to_enw(xml_root)
    raise ValueError(f"Unsupported arXiv format: {fmt}")


def get_extension(fmt):
    return {"nbib": ".nbib", "ris": ".ris", "bib": ".bib", "enw": ".enw"}.get(fmt, ".nbib")

# -*- coding: utf-8 -*-
"""
Multi-source citation downloader with format conversion.
Sources: PubMed (NCBI E-utilities), CrossRef (REST API), arXiv (Atom API).
Outputs: .nbib (PubMed only), .ris, .bib, .enw.

Usage:
  python format-converter.py --pmid 28344011
  python format-converter.py --pmid 28344011,10645439 --format ris
  python format-converter.py --doi 10.1038/nature14539 --format bib
  python format-converter.py --arxiv 1706.03762 --format ris
  python format-converter.py --query "TB-Profiler AND Bioinformatics[Journal]"
  python format-converter.py --input refs.txt
  python format-converter.py --input refs.txt --format ris
  python format-converter.py --interactive

refs.txt format:
  PMID:28344011
  DOI:10.1038/nature14539
  ARXIV:1706.03762
  QUERY:TB-Profiler AND Bioinformatics[Journal]
  AUTHOR:Dheda TITLE:drug-resistant tuberculosis
  # Lines starting with # are comments
"""

import os
import sys
import time
import json
import argparse
import xml.etree.ElementTree as ET
from urllib.request import urlopen
from urllib.parse import urlencode

from converters import (
    convert_from_medline,
    convert_from_crossref,
    convert_from_arxiv,
    get_extension,
)

EUTILS_BASE = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils"
CROSSREF_BASE = "https://api.crossref.org/works"
ARXIV_BASE = "https://export.arxiv.org/api/query"
DELAY = 0.5


# ── PubMed ──────────────────────────────────────────────────────

def esearch(query, max_results=5):
    params = {"db": "pubmed", "term": query, "retmax": max_results, "retmode": "xml"}
    url = f"{EUTILS_BASE}/esearch.fcgi?{urlencode(params)}"
    try:
        with urlopen(url, timeout=30) as resp:
            xml_data = resp.read().decode("utf-8")
        root = ET.fromstring(xml_data)
        id_list = root.find("IdList")
        if id_list is not None:
            return [e.text for e in id_list.findall("Id")]
        return []
    except Exception as e:
        print(f"  ESearch error: {e}")
        return []


def efetch_medline(pmid, retries=1):
    params = {"db": "pubmed", "id": pmid, "rettype": "medline", "retmode": "text"}
    url = f"{EUTILS_BASE}/efetch.fcgi?{urlencode(params)}"
    for attempt in range(retries):
        try:
            with urlopen(url, timeout=30) as resp:
                return resp.read().decode("utf-8")
        except Exception as e:
            if attempt == retries - 1:
                print(f"  EFetch error for PMID {pmid}: {e}")
                return None
            time.sleep(DELAY * (attempt + 1))


def download_pubmed(pmid, output_dir, fmt, retries=1):
    """Download citation by PMID. Returns (success, filename_or_error)."""
    pmid = pmid.strip()
    if not pmid:
        return False, "Empty PMID"

    print(f"  Downloading PMID: {pmid}")
    time.sleep(DELAY)
    nbib_text = efetch_medline(pmid, retries=retries)

    if not nbib_text or not nbib_text.strip():
        return False, f"PMID {pmid} not found or empty response"

    content = convert_from_medline(nbib_text, fmt)
    ext = get_extension(fmt)
    filename = f"pubmed-{pmid}{ext}"
    filepath = os.path.join(output_dir, filename)
    with open(filepath, "w", encoding="utf-8") as f:
        f.write(content)

    # Extract title for display
    for line in nbib_text.split("\n"):
        if line.startswith("TI  -"):
            print(f"  Title: {line[5:].strip()[:100]}")
            break

    print(f"  Saved: {filename} (format: {fmt})")
    return True, filename


def search_pubmed(query, output_dir, fmt, retries=1):
    print(f"  Searching: {query[:80]}...")
    time.sleep(DELAY)
    pmids = esearch(query)
    if not pmids:
        return False, f"No results for query: {query[:60]}"
    pmid = pmids[0]
    print(f"  Found PMID: {pmid}")
    return download_pubmed(pmid, output_dir, fmt, retries=retries)


# ── CrossRef ────────────────────────────────────────────────────

def download_crossref(doi, output_dir, fmt, retries=1):
    """Download citation by DOI from CrossRef. Returns (success, filename_or_error)."""
    doi = doi.strip()
    if not doi:
        return False, "Empty DOI"

    print(f"  Downloading DOI: {doi}")
    time.sleep(DELAY)
    url = f"{CROSSREF_BASE}/{doi}"
    for attempt in range(retries):
        try:
            with urlopen(url, timeout=30) as resp:
                data = json.loads(resp.read().decode("utf-8"))
            break
        except Exception as e:
            if attempt == retries - 1:
                return False, f"CrossRef API error for DOI {doi}: {e}"
            time.sleep(DELAY * (attempt + 1))

    content = convert_from_crossref(data, fmt)
    ext = get_extension(fmt)
    safe_doi = doi.replace("/", "_").replace(".", "_")[:60]
    filename = f"crossref-{safe_doi}{ext}"
    filepath = os.path.join(output_dir, filename)
    with open(filepath, "w", encoding="utf-8") as f:
        f.write(content)

    msg = data.get("message", data)
    title = msg.get("title", [])
    if title:
        print(f"  Title: {title[0][:100]}")

    print(f"  Saved: {filename} (format: {fmt})")
    return True, filename


# ── arXiv ───────────────────────────────────────────────────────

def download_arxiv(arxiv_id, output_dir, fmt, retries=1):
    """Download citation by arXiv ID. Returns (success, filename_or_error)."""
    arxiv_id = arxiv_id.strip()
    if not arxiv_id:
        return False, "Empty arXiv ID"

    print(f"  Downloading arXiv: {arxiv_id}")
    time.sleep(DELAY)
    params = {"id_list": arxiv_id, "max_results": 1}
    url = f"{ARXIV_BASE}?{urlencode(params)}"
    for attempt in range(retries):
        try:
            with urlopen(url, timeout=30) as resp:
                xml_data = resp.read().decode("utf-8")
            break
        except Exception as e:
            if attempt == retries - 1:
                return False, f"arXiv API error for ID {arxiv_id}: {e}"
            time.sleep(DELAY * (attempt + 1))

    root = ET.fromstring(xml_data)
    content = convert_from_arxiv(root, fmt)
    if not content:
        return False, f"arXiv ID {arxiv_id}: no entry found in response"

    ext = get_extension(fmt)
    safe_id = arxiv_id.replace("/", "_")[:60]
    filename = f"arxiv-{safe_id}{ext}"
    filepath = os.path.join(output_dir, filename)
    with open(filepath, "w", encoding="utf-8") as f:
        f.write(content)

    print(f"  Saved: {filename} (format: {fmt})")
    return True, filename


# ── Input parsing ───────────────────────────────────────────────

def parse_input_line(line):
    """Parse a single input line into (type, value)."""
    line = line.strip()
    if not line or line.startswith("#"):
        return None, None

    upper = line.upper()
    if upper.startswith("PMID:"):
        return "pmid", line[5:].strip()
    if upper.startswith("DOI:"):
        return "doi", line[4:].strip()
    if upper.startswith("ARXIV:"):
        return "arxiv", line[6:].strip()
    if upper.startswith("QUERY:"):
        return "query", line[6:].strip()
    if upper.startswith("AUTHOR:"):
        parts = line.split("TITLE:", 1)
        author = parts[0][7:].strip()
        title = parts[1].strip() if len(parts) > 1 else ""
        return "author_title", (author, title)

    # Default: free-text search query
    return "query", line


def process_entry(entry_type, value, output_dir, fmt, retries=1):
    if entry_type == "pmid":
        return download_pubmed(value, output_dir, fmt, retries=retries)
    elif entry_type == "doi":
        return download_crossref(value, output_dir, fmt, retries=retries)
    elif entry_type == "arxiv":
        return download_arxiv(value, output_dir, fmt, retries=retries)
    elif entry_type == "author_title":
        author, title = value
        query_parts = []
        if author:
            query_parts.append(f"{author}[Author]")
        if title:
            query_parts.append(f"{title}[Title]")
        query = " AND ".join(query_parts) if query_parts else ""
        if not query:
            return False, "Empty author and title"
        return search_pubmed(query, output_dir, fmt, retries=retries)
    elif entry_type == "query":
        return search_pubmed(value, output_dir, fmt, retries=retries)
    return False, f"Unknown entry type: {entry_type}"


def process_file(input_file, output_dir, fmt, retries=1):
    if not os.path.exists(input_file):
        print(f"Error: Input file not found: {input_file}")
        return 0, 0, [f"File not found: {input_file}"]

    with open(input_file, "r", encoding="utf-8") as f:
        lines = f.readlines()

    success, failed, errors = 0, 0, []
    for i, line in enumerate(lines, 1):
        entry_type, value = parse_input_line(line)
        if entry_type is None:
            continue
        print(f"\n[Line {i}] Processing: {line.strip()[:60]}")
        ok, result = process_entry(entry_type, value, output_dir, fmt, retries=retries)
        if ok:
            success += 1
        else:
            failed += 1
            errors.append(f"Line {i}: {result}")
    return success, failed, errors


def interactive_mode(output_dir, fmt, retries=1):
    print(f"Interactive mode (format: {fmt}) - enter references (one per line, empty line to finish):")
    print("Formats: PMID:12345 | DOI:10.xxx | ARXIV:2301.xxx | AUTHOR:Name TITLE:keywords | QUERY:...")
    print("-" * 60)

    success, failed, errors = 0, 0, []
    while True:
        try:
            line = input("> ").strip()
        except (EOFError, KeyboardInterrupt):
            break
        if not line:
            break
        entry_type, value = parse_input_line(line)
        if entry_type is None:
            continue
        ok, result = process_entry(entry_type, value, output_dir, fmt, retries=retries)
        if ok:
            success += 1
        else:
            failed += 1
            errors.append(result)
    return success, failed, errors


# ── Main ────────────────────────────────────────────────────────

def self_test():
    """Run self-check on format converter pipeline."""
    print("FORMAT CONVERTER SELF-TEST")
    print("-" * 40)

    # 1. Module import check
    try:
        from converters import convert_from_medline, convert_from_crossref, convert_from_arxiv
        print("  [OK] Module imports")
    except Exception as e:
        print(f"  [FAIL] Module imports: {e}")
        return

    # 2. PubMed endpoint (known PMID: 28344011)
    pmid = "28344011"
    print(f"  Testing PubMed (PMID {pmid})...")
    try:
        import time
        time.sleep(0.5)
        nbib_text = efetch_medline(pmid)
        if nbib_text and nbib_text.strip():
            ris_content = convert_from_medline(nbib_text, "ris")
            bib_content = convert_from_medline(nbib_text, "bib")
            enw_content = convert_from_medline(nbib_text, "enw")
            if ris_content.strip() and bib_content.strip() and enw_content.strip():
                print(f"  [OK] PubMed endpoint (RIS: {len(ris_content)}B, BibTeX: {len(bib_content)}B, ENW: {len(enw_content)}B)")
            else:
                print(f"  [FAIL] PubMed conversion produced empty output")
        else:
            print(f"  [FAIL] PubMed returned empty response for PMID {pmid}")
    except Exception as e:
        print(f"  [FAIL] PubMed endpoint: {e}")

    # 3. CrossRef endpoint (known DOI)
    doi = "10.1038/nature14539"
    print(f"  Testing CrossRef (DOI {doi})...")
    try:
        from urllib.request import urlopen
        import json
        time.sleep(0.5)
        url = f"https://api.crossref.org/works/{doi}"
        with urlopen(url, timeout=30) as resp:
            data = json.loads(resp.read().decode("utf-8"))
        ris_content = convert_from_crossref(data, "ris")
        bib_content = convert_from_crossref(data, "bib")
        enw_content = convert_from_crossref(data, "enw")
        if ris_content.strip() and bib_content.strip() and enw_content.strip():
            print(f"  [OK] CrossRef endpoint (RIS: {len(ris_content)}B, BibTeX: {len(bib_content)}B, ENW: {len(enw_content)}B)")
        else:
            print(f"  [FAIL] CrossRef conversion produced empty output")
    except Exception as e:
        print(f"  [FAIL] CrossRef endpoint: {e}")

    print("-" * 40)
    print("Self-test complete.")


def main():
    parser = argparse.ArgumentParser(
        description="Multi-source citation downloader with format conversion (.nbib/.ris/.bib)",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog="""
Examples:
  %(prog)s --pmid 28344011
  %(prog)s --pmid 28344011,10645439 --format ris
  %(prog)s --doi 10.1038/nature14539 --format bib
  %(prog)s --doi 10.1038/nature14539,10.1038/s41586-020-2649-2 --format ris
  %(prog)s --arxiv 1706.03762 --format bib
  %(prog)s --arxiv 1706.03762,2302.13971 --format ris
  %(prog)s --query "TB-Profiler AND Bioinformatics[Journal]"
  %(prog)s --input refs.txt
  %(prog)s --input refs.txt --format ris
  %(prog)s --interactive

refs.txt format:
  PMID:28344011
  DOI:10.1038/nature14539
  ARXIV:1706.03762
  QUERY:TB-Profiler AND Bioinformatics[Journal]
  AUTHOR:Dheda TITLE:drug-resistant tuberculosis
  # Lines starting with # are comments
        """,
    )
    parser.add_argument("--pmid", help="PMID(s), comma-separated")
    parser.add_argument("--doi", help="DOI(s), comma-separated")
    parser.add_argument("--arxiv", help="arXiv ID(s), comma-separated")
    parser.add_argument("--author", help="Author name for PubMed search")
    parser.add_argument("--title", help="Title keywords for PubMed search")
    parser.add_argument("--query", help="PubMed search query")
    parser.add_argument("--input", help="Input file with references")
    parser.add_argument("--interactive", action="store_true", help="Interactive mode")
    parser.add_argument(
        "--format", choices=["nbib", "ris", "bib", "enw"], default="nbib",
        help="Output format: nbib (default, MEDLINE), ris (EndNote/Zotero), bib (BibTeX/LaTeX), enw (EndNote tagged)",
    )
    parser.add_argument(
        "--output", default="./references/",
        help="Output directory (default: ./references/)",
    )
    parser.add_argument("--version", action="version", version="format-converter 1.0.0")
    parser.add_argument("--test", action="store_true", help="Run self-test on format converter pipeline")
    parser.add_argument("--retry", type=int, default=1, help="Retry count for HTTP calls")
    parser.add_argument("--preflight", action="store_true", help="Run connectivity check on API endpoints")

    args = parser.parse_args()

    if args.test:
        self_test()
        return

    if args.preflight:
        import sys as _sys
        _sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
        from preflight import check_endpoints
        results = check_endpoints()
        # Print report
        print("PRE-FLIGHT REPORT")
        all_ok = True
        for name, info in results.items():
            status = "OK" if info["ok"] else "FAIL"
            detail = f"({info['time']:.1f}s)" if info["ok"] else f"({info['error']})"
            print(f"  {name:25s}: {status} {detail}")
            if not info["ok"]:
                all_ok = False
        reachable = sum(1 for v in results.values() if v["ok"])
        total = len(results)
        print(f"  {reachable}/{total} endpoints reachable.")
        if not all_ok:
            print("  Affected: format-converter downloads for unreachable endpoints (MCP tools unaffected).")
            _sys.exit(1)
        return

    if not any([args.pmid, args.doi, args.arxiv, args.author, args.query, args.input, args.interactive]):
        parser.error("Specify at least one input method")

    fmt = args.format
    output_dir = os.path.abspath(args.output)
    if not os.path.exists(output_dir):
        os.makedirs(output_dir)
        print(f"Created output directory: {output_dir}")

    print(f"Output directory: {output_dir}")
    print(f"Format: {fmt}")
    print("=" * 60)

    total_success, total_failed, all_errors = 0, 0, []

    def handle_list(ids_str, handler, retries=1):
        nonlocal total_success, total_failed
        ids = [x.strip() for x in ids_str.split(",") if x.strip()]
        for item_id in ids:
            print(f"\nProcessing: {item_id}")
            ok, result = handler(item_id, output_dir, fmt, retries=retries)
            if ok:
                total_success += 1
            else:
                total_failed += 1
                all_errors.append(result)

    # --pmid
    if args.pmid:
        handle_list(args.pmid, download_pubmed, retries=args.retry)

    # --doi
    if args.doi:
        if fmt == "nbib":
            print("Warning: CrossRef does not provide .nbib (MEDLINE) format. Falling back to .ris")
            fmt = "ris"
        handle_list(args.doi, download_crossref, retries=args.retry)

    # --arxiv
    if args.arxiv:
        if fmt == "nbib":
            print("Warning: arXiv does not provide .nbib (MEDLINE) format. Falling back to .ris")
            fmt = "ris"
        handle_list(args.arxiv, download_arxiv, retries=args.retry)

    # --author / --title / --query
    if args.author or args.title or args.query:
        if args.query:
            query = args.query
        else:
            query_parts = []
            if args.author:
                query_parts.append(f"{args.author}[Author]")
            if args.title:
                query_parts.append(f"{args.title}[Title]")
            query = " AND ".join(query_parts)
        print(f"\nProcessing search: {query}")
        ok, result = search_pubmed(query, output_dir, fmt, retries=args.retry)
        if ok:
            total_success += 1
        else:
            total_failed += 1
            all_errors.append(result)

    # --input
    if args.input:
        print(f"\nProcessing file: {args.input}")
        s, f, e = process_file(args.input, output_dir, fmt, retries=args.retry)
        total_success += s
        total_failed += f
        all_errors.extend(e)

    # --interactive
    if args.interactive:
        s, f, e = interactive_mode(output_dir, fmt, retries=args.retry)
        total_success += s
        total_failed += f
        all_errors.extend(e)

    # Summary
    print("\n" + "=" * 60)
    print("SUMMARY")
    print(f"  Success: {total_success}")
    print(f"  Failed:  {total_failed}")
    if all_errors:
        print("  Errors:")
        for err in all_errors:
            print(f"    - {err}")
    print(f"  Output:  {output_dir}")
    print("=" * 60)


if __name__ == "__main__":
    main()

# -*- coding: utf-8 -*-
"""
Pre-flight API endpoint connectivity checker.

Verifies that the 3 direct API endpoints used by format-converter.py are
reachable. Uses urllib.request.urlopen (stdlib) for consistency with the
existing codebase.

Usage:
  python preflight.py                       # run check and print report
  python -c "from preflight import check_endpoints; print(check_endpoints())"
"""

import sys
import time
from urllib.request import urlopen

ENDPOINTS = [
    {
        "name": "PubMed E-utilities",
        "url": "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=pubmed&retmax=1",
        "timeout": 10,
        "affected": "PubMed downloads in format-converter",
    },
    {
        "name": "CrossRef REST",
        "url": "https://api.crossref.org/works/10.1038/nature14539",
        "timeout": 10,
        "affected": "CrossRef/DOI downloads in format-converter",
        "expect_status": 200,
    },
    {
        "name": "arXiv API",
        "url": "https://export.arxiv.org/api/query?id_list=1706.03762&max_results=1",
        "timeout": 10,
        "affected": "arXiv downloads in format-converter (MCP search_arxiv unaffected)",
    },
]


def check_single(name, url, timeout, expect_status=None):
    """Check a single endpoint. Returns (ok, elapsed, error_message_or_None)."""
    start = time.perf_counter()
    try:
        with urlopen(url, timeout=timeout) as resp:
            status = resp.status
            _ = resp.read()  # consume body to confirm full response
        elapsed = time.perf_counter() - start
        if expect_status is not None and status != expect_status:
            return False, elapsed, f"unexpected HTTP {status} (expected {expect_status})"
        return True, elapsed, None
    except Exception as e:
        elapsed = time.perf_counter() - start
        # Produce a short error label: prefer the os-error string over full traceback
        error_str = str(e)
        if "time" in error_str.lower() and "out" in error_str.lower():
            error_str = "timeout after {}s".format(timeout)
        elif hasattr(e, "reason") and e.reason is not None:
            error_str = str(e.reason)
        elif hasattr(e, "code"):
            error_str = "HTTP {}".format(e.code)
        return False, elapsed, error_str


def check_endpoints():
    """Check all configured endpoints.

    Returns:
        dict[str, dict]: {name: {"ok": bool, "time": float, "error": str|None}}
    """
    results = {}
    for ep in ENDPOINTS:
        ok, elapsed, error = check_single(
            ep["name"],
            ep["url"],
            ep["timeout"],
            ep.get("expect_status"),
        )
        results[ep["name"]] = {
            "ok": ok,
            "time": elapsed,
            "error": error,
        }
    return results


def print_report(results):
    """Print a human-readable pre-flight report."""
    ok_count = sum(1 for v in results.values() if v["ok"])
    total = len(results)

    print("PRE-FLIGHT REPORT")
    for name, info in results.items():
        status = "OK" if info["ok"] else "FAIL"
        if info["ok"]:
            extra = ""
        else:
            extra = " ({})".format(info["error"])
        print("  {:20s}: {} ({:.1f}s){}".format(name, status, info["time"], extra))

    print("  {}/{} endpoints reachable.".format(ok_count, total))

    if ok_count < total:
        print("  Affected:")
        for ep in ENDPOINTS:
            name = ep["name"]
            info = results[name]
            if not info["ok"]:
                print("    - {}: {}".format(name, ep["affected"]))

    return ok_count == total


def main():
    results = check_endpoints()
    all_ok = print_report(results)
    sys.exit(0 if all_ok else 1)


if __name__ == "__main__":
    main()

---
name: nature-academic-search
description: >-
  Multi-source literature search, citation verification, strict independent other-citation
  audits, article-level citation metric tables, influential citer profiling with
  citation-context extraction, MeSH search strategy, citation file management
  (.nbib/.ris/.bib conversion), and reference management (BibTeX, related articles,
  ID conversion) via MCP tools (PubMed, CrossRef, arXiv, Scopus, ScienceDirect).
  Use for coordinated literature workflows beyond one MCP call, including 文献检索、
  查文献、找文献、文献综述检索、查论文、引文核对、参考文献管理、文献去重、
  严格他引、他引判定、排除自引、谁引用了我的文章、引用我的文章的人有没有大牛、
  院士引用、校长引用、院长引用、杰青引用、长江学者引用、Fellow引用、文章引用表、
  指定文章引用数、严格他引数、整理成表格.
---

# Academic Search — Router

This skill is split into two layers:

- A **static layer** under `static/` that holds versioned, reusable content fragments (the MCP tool inventory and shared modules, and source routing plus operational rules).
- A **dynamic layer** (this file plus `manifest.yaml`) that detects which workflow the user needs and loads that workflow, reaching for shared modules and scripts only when a step needs them.

Do not try to apply the search logic from memory or from this router. Always load fragments from disk as described below.

## Routing protocol

Follow these five steps every time the skill is invoked.

### 1. Load the manifest and the core layer

Read [manifest.yaml](manifest.yaml). It declares the `workflow` axis, the allowed values, and the file paths each value maps to.

Also read every file listed under `always_load`:

- `static/core/tools.md` — the MCP tool inventory (core search, extended search, PubMed utilities) and the shared-module map.
- `static/core/routing-and-ops.md` — the T1→T2→T3 source routing quick guide, environment setup, error handling, and limitations.

### 2. Detect the workflow

Map the user's need to one or more `workflow` values:

- `multi-source-search` — find literature across sources.
- `citation-verification` — verify citations extracted from a document.
- `mesh-strategy` — build a MeSH/PubMed search strategy.
- `citation-file-mgmt` — convert/manage `.nbib`/`.ris`/`.bib` files.
- `reference-mgmt` — BibTeX, related-article discovery, ID conversion.
- `strict-other-citation-impact-audit` — determine strict independent other-citations, build article-level citation metric tables, identify high-profile citers (academy members, presidents/deans, talent-award holders, fellows, field leaders), and extract how they cited the target paper.

A combined request (for example search then export) may need more than one. State the detected workflow(s) in one short line before proceeding.

### 3. Load the matching workflow fragment(s)

Read the file mapped for each detected workflow (under `references/workflows/`). Do **not** read every workflow. Each workflow file links to the shared modules it needs.

### 4. Run the workflow using the loaded material

Apply the loaded material in this order:

1. Core tools and routing (`core/tools.md`, `core/routing-and-ops.md`) — which MCP tool for which need, and the T1→T2→T3 fallback chain that is the standard execution order across all workflows.
2. The workflow fragment — its specific steps.
3. Shared modules and scripts on demand (dedup, citation parser, search strategy, RIS/BibTeX format, format converter).

Report specific tool failures and continue with remaining tools; broaden terms when there are no results; fall back to manual generation from MCP-fetched metadata if a script fails twice.

### 5. Reach for references only when needed

The files under `references/` (and `scripts/`) are deep references, not defaults. Open them on demand per the `references.on_demand` table in the manifest — for example `references/source-tiers.md` for the full reliability classification, `references/dedup-engine.md` / `references/citation-parser.md` / `references/search-strategy.md` / `references/ris-bibtex-format.md` for the shared modules, and `scripts/academic_search.py` (no-MCP fallback discovery search) / `scripts/format-converter.py` / `scripts/preflight.py` for the tooling.

## Why this split

- The static layer is versioned and reviewable; the workflow files and shared modules were already factored this way.
- The dynamic layer keeps each invocation cheap: only the workflow the user needs enters context, instead of all six plus every module.
- The router itself is short on purpose. Update fragments and references, not this file, when adding scope.
- This structure mirrors the other nature-* skills (`nature-writing`, `nature-polishing`, `nature-reader`, `nature-paper2ppt`, `nature-figure`, `nature-citation`, `nature-response`, `nature-data`).

# Source routing and operations

## Source routing

See [Source Tiers & Reliability](../../references/source-tiers.md) for the complete reliability classification and fallback routing rules. The T1→T2→T3 fallback chain is the standard execution order across all workflows.

Quick guide:

| User need | Primary (T1) | Secondary (T2) | Last Resort (T3) |
|-----------|-------------|-----------------|-------------------|
| Medical / clinical | PubMed | Semantic Scholar | Google Scholar |
| Cross-disciplinary | CrossRef | Semantic Scholar | Scopus |
| Preprints / CS / physics | arXiv | bioRxiv / medRxiv | — |
| Exhaustive review | PubMed + CrossRef + arXiv | Semantic Scholar + bioRxiv/medRxiv | WoS / Scopus |
| Citation count sensitive | Semantic Scholar | CrossRef | — |
| Chinese literature | — | — | CNKI / 万方 (manual) |

## Environment setup

### API keys (optional but recommended)

| Service | Env Var | Register At | Free Tier |
|---------|---------|-------------|-----------|
| Semantic Scholar | `SEMANTIC_SCHOLAR_API_KEY` | [api.semanticscholar.org](https://api.semanticscholar.org/) | 100 req/s with key (1/s without) |
| NCBI E-utilities | `NCBI_API_KEY` | [ncbi.nlm.nih.gov/account](https://www.ncbi.nlm.nih.gov/account/) | 10 req/s with key (3/s without) |
| Elsevier / Scopus / ScienceDirect | pybliometrics config | [dev.elsevier.com](https://dev.elsevier.com/) | Depends on API entitlement |

Set Semantic Scholar / NCBI keys via `export` or `.env` file. Elsevier keys are read from the local pybliometrics config, normally `~/.config/pybliometrics.cfg`; do not copy API keys into this plugin.

### Proxy (if behind firewall)

```bash
export http_proxy=http://127.0.0.1:7890
export https_proxy=http://127.0.0.1:7890
```

### Pre-flight check

```bash
python scripts/preflight.py
```

Run before batch operations to verify API endpoints are reachable.

### Format converter dependencies

The format converter (`scripts/format-converter.py`) uses Python stdlib only — no extra dependencies. Run `python scripts/format-converter.py --test` to verify the conversion pipeline.

### No-MCP fallback (standalone search)

When the MCP server is not mounted (plain CLI use, skill auto-discovery, CI), the search workflow still runs via two stdlib-only scripts that hit public HTTP APIs directly — the same pattern `nature-citation/scripts/nature_citation.py` uses for CrossRef:

- **Discovery** — `scripts/academic_search.py` queries OpenAlex (free, no API key). OpenAlex indexes CrossRef, PubMed and arXiv-deposited works, so one endpoint covers journals and preprints for keyword/author search, with relevance re-ranking and author disambiguation (`--affiliation` / `--orcid` / `--list-authors`). Returns ranked JSON (title, DOI, authors, year, citations, abstract).
- **Download / convert** — `scripts/format-converter.py` turns the chosen DOIs/PMIDs/arXiv IDs into `.ris`/`.bib`/`.enw`/`.nbib` (CrossRef + PubMed + arXiv, also stdlib-only).

```bash
# discover, then export the picks
python scripts/academic_search.py "graph neural network potentials" --limit 10 --sort cited_by_count --mailto [email protected]
python scripts/format-converter.py --doi 10.1103/physrevlett.120.143001 --format ris
```

Be polite to the OpenAlex pool: pass `--mailto` or set `OPENALEX_MAILTO` / `CROSSREF_MAILTO`. Each script reports per-source failures (HTTP 429, timeout, network) on stderr and exits non-zero, so a caller treats each source independently and continues with another tool. This fallback covers the same T1→T2→T3 sources for *discovery* via OpenAlex; the MCP path remains preferred when available (per-source tool selection, Semantic Scholar / Scopus providers).

### MCP server runtime

Use uv to start the MCP server in an isolated dependency environment:

```bash
uv run --no-project --directory <mcp-server> --with "mcp>=1.0.0,<2.0.0" --with "requests>=2.28.0,<3.0.0" --with "toml>=0.10.2,<2.0.0" --with "lxml>=4.9.0,<6.0.0" --with "pybliometrics>=4.4.1,<5.0.0" python academic_search_server.py
```

`search_papers` defaults to CrossRef, PubMed, and arXiv. Scopus / ScienceDirect are opt-in providers: include `scopus` / `sciencedirect` in `sources`, or call their dedicated tools. These calls use the local pybliometrics config at `~/.config/pybliometrics.cfg` and may consume Elsevier API quota.

## Error handling

- **MCP tool unavailable**: report specific failure, continue with remaining tools. If the whole MCP server is absent, switch to the No-MCP fallback (`scripts/academic_search.py` + `scripts/format-converter.py`) above.
- **No results**: broaden terms, try alternative sources, suggest user refine query.
- **Script failure (2x)**: fall back to manual generation from MCP-fetched metadata.

## Limitations

- Google Scholar and Semantic Scholar are scraped (not API-backed) — results may vary.
- Chinese literature (CNKI / 万方) not indexed by CrossRef or PubMed.
- Citation counts may be delayed (CrossRef updates monthly).

# MCP tools and shared modules

Multi-source literature search, citation verification, citation format conversion, and reference management via MCP tools.

## MCP tools

### Core search

| Tool | Source | Best For |
|------|--------|----------|
| `search_papers` | academic-search MCP | Default concurrent search across CrossRef, PubMed, arXiv; accepts opt-in Scopus / ScienceDirect sources |
| `get_paper_by_id` | academic-search MCP | DOI / PMID / arXiv ID details |
| `get_citation` | academic-search MCP | DOI-based formatted citation |
| `lookup_mesh` | academic-search MCP | MeSH term exploration |

### Scopus / ScienceDirect tools

| Tool | Source | Best For |
|------|--------|----------|
| `search_scopus` | academic-search MCP | Scopus advanced document search |
| `get_scopus_abstract` | academic-search MCP | Scopus abstract and document metadata |
| `get_scopus_citation_overview` | academic-search MCP | Scopus citation overview |
| `search_scopus_authors` / `get_scopus_author` | academic-search MCP | Author profile search and retrieval |
| `search_scopus_affiliations` / `get_scopus_affiliation` | academic-search MCP | Affiliation search and retrieval |
| `search_scopus_serial_titles` / `get_scopus_serial_title` | academic-search MCP | Journal/source metadata |
| `get_scopus_plumx_metrics` | academic-search MCP | PlumX metrics |
| `search_sciencedirect` | academic-search MCP | ScienceDirect article search |
| `get_sciencedirect_article_metadata` | academic-search MCP | ScienceDirect article metadata |

### Extended search

| Tool | Source | Best For |
|------|--------|----------|
| `search_google_scholar` | paper-search MCP | Broad academic search (scraped) |
| `search_semantic_scholar` | paper-search MCP | Citation graph, field-of-study filters |
| `search_biorxiv` | paper-search MCP | Biology preprints |
| `search_medrxiv` | paper-search MCP | Medical preprints |
| `search_webofscience` | paper-search MCP | Curated index, citation reports |
| `search_scopus` | paper-search MCP | Broad scholarly database |

### PubMed utilities

| Tool | Purpose |
|------|---------|
| `pubmed_fetch_articles` | Full metadata by PMID |
| `pubmed_find_related` | Related article discovery |
| `pubmed_format_citations` | APA / MLA / BibTeX / RIS formatting |
| `pubmed_convert_ids` | DOI ↔ PMID ↔ PMCID conversion |
| `pubmed_lookup_mesh` | MeSH term exploration and hierarchy |
| `pubmed_lookup_citation` | Bibliographic citation → PMID lookup |

## Shared modules

| Module | Purpose |
|--------|---------|
| [Dedup Engine](../../references/dedup-engine.md) | Unified deduplication (WFs 1, 2, 5a) |
| [Citation Parser](../../references/citation-parser.md) | Extract citations from documents (WF 2) |
| [Search Strategy](../../references/search-strategy.md) | Query construction, source selection, ranking |
| [RIS/BibTeX Format](../../references/ris-bibtex-format.md) | Format specifications and field mappings |
| [Format Converter](../../scripts/format-converter.py) | Multi-source .nbib/.ris/.bib downloader |