scieloorg · samuelveigarangel · Apr 29, 2026 · Apr 29, 2026 · Apr 29, 2026 · Apr 29, 2026
diff --git a/.env.example b/.env.example
@@ -0,0 +1,12 @@
+PROCESSING_SETTINGS_FILE=
+ARTICLEMETA_THRIFTSERVER=
+ARTICLEMETA_ADMINTOKEN=
+RATCHET_THRIFTSERVER=
+ACCESSSTATS_THRIFTSERVER=
+CITEDBY_THRIFTSERVER=
+PUBLICATIONSTATS_THRIFTSERVER=
+SOLR_SEARCH_SCIELO_ORG=
+SOLR_SEARCH_SCIELO_ORG_INDEX=
+PUBLICATIONSTATS_TIMEOUT_MS=60000
+EXIT_ON_FAILURE=true
+SLACK_WEBHOOK_URL=
diff --git a/.github/workflows/master-quality.yml b/.github/workflows/master-quality.yml
@@ -0,0 +1,26 @@
+name: Master Quality
+
+on:
+  push:
+    branches:
+      - codex/python3-14-migration
+
+jobs:
+  sonar:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: SonarQube Scan
+        uses: SonarSource/sonarqube-scan-action@v6
+        env:
+          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+          SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }}
+
+      - name: Quality Gate
+        uses: SonarSource/sonarqube-quality-gate-action@v1
+        env:
+          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -0,0 +1,117 @@
+name: Release
+
+on:
+  push:
+    tags:
+      - 'v*'
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: read
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set version
+        run: echo "VERSION=${GITHUB_REF_NAME}" >> $GITHUB_ENV
+
+      - name: Docker Login
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USER }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Build image
+        run: |
+          docker build \
+            -t infrascielo/processing:${VERSION} \
+            -t infrascielo/processing:latest \
+            .
+
+      # 🔐 Scan único (policy)
+      - name: Trivy Image Scan
+        uses: aquasecurity/trivy-action@v0.36.0
+        with:
+          image-ref: infrascielo/processing:${{ env.VERSION }}
+          severity: HIGH,CRITICAL
+          exit-code: 0
+
+      - name: Install Trivy CLI
+        run: |
+          curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sudo sh -s -- -b /usr/local/bin
+
+      # 📄 Relatório (evidência)
+      - name: Trivy Report
+        run: |
+          trivy image \
+            --scanners vuln \
+            --severity HIGH,CRITICAL \
+            --format table \
+            --output trivy-report.txt \
+            infrascielo/processing:${VERSION}
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: trivy-report
+          path: trivy-report.txt
+
+      # 📦 SBOM
+      - name: Generate SBOM (CycloneDX)
+        run: |
+          trivy image \
+            --scanners vuln \
+            --format cyclonedx \
+            --output sbom-${VERSION}.json \
+            infrascielo/processing:${VERSION}
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: sbom-${{ env.VERSION }}
+          path: sbom-${{ env.VERSION }}.json
+
+      - name: Push image
+        run: |
+          docker push infrascielo/processing:${VERSION}
+          docker push infrascielo/processing:latest
+
+      - name: Push image
+        run: |
+          docker push infrascielo/processing:${VERSION}
+          docker push infrascielo/processing:latest
+
+      - name: Get image digest
+        run: |
+          DIGEST=$(docker inspect --format='{{index .RepoDigests 0}}' infrascielo/processing:${VERSION})
+          echo "IMAGE_DIGEST=${DIGEST}" >> $GITHUB_ENV
+
+      - name: Install Cosign
+        uses: sigstore/cosign-installer@v3
+
+      - name: Sign image with Cosign
+        env:
+          COSIGN_EXPERIMENTAL: "1"
+          COSIGN_YES: "true"
+        run: |
+          cosign sign ${IMAGE_DIGEST}
+
+      - name: Verify image signature
+        env:
+          COSIGN_EXPERIMENTAL: "1"
+        run: |
+          cosign verify \
+            --certificate-oidc-issuer https://token.actions.githubusercontent.com \
+            --certificate-identity-regexp "https://github.com/${{ github.repository }}/*" \
+            ${IMAGE_DIGEST}
+
+      - name: Attach SBOM attestation
+        env:
+          COSIGN_EXPERIMENTAL: "1"
+          COSIGN_YES: "true"
+        run: |
+          cosign attest \
+            --predicate sbom-${VERSION}.json \
+            --type cyclonedx \
+            ${IMAGE_DIGEST}
diff --git a/.gitignore b/.gitignore
@@ -1,4 +1,8 @@
 .DS_Store
+*.swp
+.env
+.env.*
+!.env.example
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]
@@ -57,5 +61,11 @@ docs/_build/
 # PyBuilder
 target/
 
+# Runtime processing workspace
+tmp_*/
+network/
+var/
+k8s/*secret.yaml
+
 # pip source directory
 src/
diff --git a/Dockerfile b/Dockerfile
@@ -0,0 +1,40 @@
+FROM python:3.14-slim
+
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    PIP_DISABLE_PIP_VERSION_CHECK=1 \
+    PROCESSING_SETTINGS_FILE=/app/config.ini \
+    DOCKER_ENV=1 \
+    WORK_DIR=/app \
+    LOG_DIR=/var/log/processing \
+    TABS_DIR=/var/www/static_scielo_org/tabs \
+    PYTHONPATH=/app
+
+WORKDIR /app
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends \
+        build-essential \
+        curl \
+        git \
+        libxml2-dev \
+        libxslt1-dev \
+        zip \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY requirements.txt setup.py ./
+RUN pip install --upgrade pip setuptools wheel \
+    && pip install -r requirements.txt \
+    && pip install --no-deps \
+        accessstatsapi==1.2.1 \
+        articlemetaapi==1.26.7 \
+        citedbyapi==1.11.3 \
+        publicationstatsapi==1.2.2 \
+        packtools==2.6.4
+
+COPY . .
+RUN cp config.ini-TEMPLATE config.ini \
+    && mkdir -p /var/log/processing /var/www/static_scielo_org/tabs \
+    && pip install --no-deps -e .
+
+ENTRYPOINT ["/bin/bash", "/app/run.sh"]
diff --git a/README.md b/README.md
@@ -5,7 +5,76 @@ e também para o envio de dados à parceiros.
 
 
 ## Requisitos:
-* Python 2.7
+* Python 3.14
+
+## Docker
+
+Para instalar as dependências e executar a suíte de testes:
+
+```bash
+docker compose run --rm tests
+```
+
+Para executar o processamento via Docker, configure as variáveis em um arquivo
+`.env` local, baseado em `.env.example`. O `.env` não deve ser versionado.
+
+```bash
+cp .env.example .env
+```
+
+Preencha no `.env` os valores equivalentes ao antigo `config.ini`:
+
+```ini
+ARTICLEMETA_THRIFTSERVER=
+ARTICLEMETA_ADMINTOKEN=
+RATCHET_THRIFTSERVER=
+ACCESSSTATS_THRIFTSERVER=
+CITEDBY_THRIFTSERVER=
+PUBLICATIONSTATS_THRIFTSERVER=
+SOLR_SEARCH_SCIELO_ORG=
+SOLR_SEARCH_SCIELO_ORG_INDEX=
+PUBLICATIONSTATS_TIMEOUT_MS=60000
+SLACK_WEBHOOK_URL=
+```
+
+Depois execute:
+
+```bash
+docker compose run --rm processing "scl-BR"
+```
+
+Os logs ficam persistidos em `var/log/processing` e os arquivos ZIP gerados em
+`var/tabs`.
+
+## Kubernetes / Argo
+
+O agendamento para Argo Workflows está em `k8s/argo-cronworkflow.yaml` com o cron:
+
+```text
+0 3 1,8,15,22 1-12 *
+```
+
+O workflow executa as coleções sequencialmente. Quando uma coleção falha, o
+script envia a notificação de erro e continua para a próxima; ao final, o job
+termina com sucesso para não interromper as próximas execuções agendadas.
+
+Os valores sensíveis devem ser criados em um Secret do Kubernetes chamado
+`processing-env`, baseado em `k8s/processing-env.secret.example.yaml`. Não
+versione o Secret real.
+
+Volumes esperados:
+
+```text
+processing-data -> /var/www/static_scielo_org/tabs
+processing-logs -> /var/log/processing
+```
+
+Aplicação:
+
+```bash
+kubectl apply -f k8s/processing-env.secret.yaml
+kubectl apply -f k8s/argo-cronworkflow.yaml
+```
 
 
 ## Exportação de dados ao DOAJ

diff --git a/accesses/documents_by_journals.py b/accesses/documents_by_journals.py
@@ -51,35 +51,35 @@ def __init__(self, collection, issns=None, output_file=None):
         self.issns = issns
         self.output_file = codecs.open(output_file, 'w', encoding='utf-8') if output_file else output_file
         header = []
-        header.append(u"extraction date")
-        header.append(u"study unit")
-        header.append(u"collection")
-        header.append(u"ISSN SciELO")
-        header.append(u"ISSN\'s")
-        header.append(u"title at SciELO")
-        header.append(u"title thematic areas")
+        header.append("extraction date")
+        header.append("study unit")
+        header.append("collection")
+        header.append("ISSN SciELO")
+        header.append("ISSN\'s")
+        header.append("title at SciELO")
+        header.append("title thematic areas")
         for area in choices.THEMATIC_AREAS:
-            header.append(u"title is %s" % area.lower())
-        header.append(u"title is multidisciplinary")
-        header.append(u"title current status")
-        header.append(u"publishing year")
-        header.append(u"accesses year")
-        header.append(u"accesses to html")
-        header.append(u"accesses to abstract")
-        header.append(u"accesses to pdf")
-        header.append(u"accesses to epdf")
-        header.append(u"total accesses")
-
-        self.write(u','.join([u'"%s"' % i.replace(u'"', u'""') for i in header]))
+            header.append("title is %s" % area.lower())
+        header.append("title is multidisciplinary")
+        header.append("title current status")
+        header.append("publishing year")
+        header.append("accesses year")
+        header.append("accesses to html")
+        header.append("accesses to abstract")
+        header.append("accesses to pdf")
+        header.append("accesses to epdf")
+        header.append("total accesses")
+
+        self.write(','.join(['"%s"' % i.replace('"', '""') for i in header]))
 
     def write(self, line):
         if not self.output_file:
-            print(line.encode('utf-8'))
+            print(line)
         else:
             self.output_file.write('%s\r\n' % line)
 
     def run(self):
-        for item in self.items():
+        for item in list(self.items()):
             self.write(item)
         logger.info('Export finished')
 
@@ -103,17 +103,17 @@ def fmt_csv(self, data):
 
         line = []
         line.append(datetime.datetime.now().isoformat()[0:10])
-        line.append(u'journal')
+        line.append('journal')
         line.append(data.collection_acronym)
         line.append(data.scielo_issn)
-        line.append(u';'.join(issns))
+        line.append(';'.join(issns))
         line.append(data.title)
-        line.append(u';'.join(data.subject_areas or []))
+        line.append(';'.join(data.subject_areas or []))
         for area in choices.THEMATIC_AREAS:
             if area.lower() in [i.lower() for i in data.subject_areas or []]:
-                line.append(u'1')
+                line.append('1')
             else:
-                line.append(u'0')
+                line.append('0')
         line.append('1' if len(data.subject_areas or []) > 2 else '0')
         line.append(data.current_status)