Arabic + Hebrew SOTA stack, SadeedDiac-25 benchmark win, papers

.github/dependabot.yml

@@ -0,0 +1,10 @@
+version: 2
+updates:
+  - package-ecosystem: github-actions
+    directory: "/"
+    schedule:
+      interval: weekly
+  - package-ecosystem: bundler
+    directory: "/"
+    schedule:
+      interval: weekly

.github/workflows/codeql.yml

@@ -0,0 +1,27 @@
+name: codeql
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+  schedule:
+    - cron: "0 0 * * 0"  # weekly
+
+permissions:
+  actions: read
+  contents: read
+  security-events: write
+
+jobs:
+  analyze:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ruby]
+    steps:
+      - uses: actions/checkout@v7
+      - uses: github/codeql-action/init@v3
+        with:
+          languages: ${{ matrix.language }}
+      - uses: github/codeql-action/analyze@v3

.github/workflows/python-arabic.yml

@@ -6,76 +6,82 @@
   pull_request:
 
 jobs:
+  lint:
+    runs-on: ubuntu-latest
+    continue-on-error: true  # 36 pre-existing violations; see TODO.complete/08-ruff-rababa-python.md
+    steps:
+      - uses: actions/checkout@v7
+      - uses: actions/setup-python@v7
+        with:
+          python-version: "3.11"
+      - run: pip install ruff
+      - run: ruff check python/
+      - run: ruff format --check python/
+
   infer:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.6', '3.7', '3.8', '3.9']
+        python-version: ["3.9"]
 
     steps:
-    - uses: actions/checkout@v2
-    - uses: actions/setup-python@v2
-      with:
-        python-version: ${{ matrix.python-version }}
+      - uses: actions/checkout@v7
+      - uses: actions/setup-python@v7
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: pip
+          cache-dependency-path: python/arabic/requirements.txt
 
-    - uses: actions/cache@v2
-      with:
-        path: ${{ env.pythonLocation }}
-        key: ${{ env.pythonLocation }}-${{ hashFiles('python/arabic/setup.py') }}-${{ hashFiles('python/arabic/requirements.txt') }}
+      - name: Install requirements
+        working-directory: ./python/arabic
+        run: |
+          pip install --upgrade --upgrade-strategy eager -r requirements.txt -e .
 
-    - name: Install requirements
-      working-directory: ./python/arabic
-      run: |
-        pip install --upgrade --upgrade-strategy eager -r requirements.txt -e .
+      - name: Download PyTorch model
+        working-directory: ./python/arabic
+        run: |
+          curl -sSL https://github.com/secryst/rababa-models/releases/download/0.1/2000000-snapshot.pt \
+            -o log_dir/CA_MSA.base.cbhg/models/2000000-snapshot.pt
 
-    - name: Download PyTorch model
-      working-directory: ./python/arabic
-      run: |
-        curl -sSL https://github.com/secryst/rababa-models/releases/download/0.1/2000000-snapshot.pt \
-          -o log_dir/CA_MSA.base.cbhg/models/2000000-snapshot.pt
-
-    - name: Run diacriticization
-      working-directory: ./python/arabic
-      run: |
-        python diacritize.py --model_kind "cbhg" --config config/cbhg.yml --text 'قطر'
+      - name: Run diacriticization
+        working-directory: ./python/arabic
+        run: |
+          python diacritize.py --model_kind "cbhg" --config config/cbhg.yml --text 'قطر'
 
   train:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.6', '3.7', '3.8', '3.9']
+        python-version: ["3.9"]
 
     steps:
-    - uses: actions/checkout@v2
-    - uses: actions/setup-python@v2
-      with:
-        python-version: ${{ matrix.python-version }}
-
-    - uses: actions/cache@v2
-      with:
-        path: ${{ env.pythonLocation }}
-        key: ${{ env.pythonLocation }}-${{ hashFiles('python/setup.py') }}-${{ hashFiles('python/requirements.txt') }}
+      - uses: actions/checkout@v7
+      - uses: actions/setup-python@v7
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: pip
+          cache-dependency-path: python/arabic/requirements.txt
 
-    - name: Install requirements
-      working-directory: ./python/arabic
-      run: |
-        pip install --upgrade --upgrade-strategy eager -r requirements.txt -e .
+      - name: Install requirements
+        working-directory: ./python/arabic
+        run: |
+          pip install --upgrade --upgrade-strategy eager -r requirements.txt -e .
 
-    - name: Prepare dataset
-      working-directory: ./python/arabic
-      run: |
-        mkdir -p data/CA_MSA
-        touch data/CA_MSA/{eval,train,test}.csv
-        cd data
-        curl -sSL https://github.com/interscript/rababa-tashkeela/archive/refs/tags/v1.0.zip -o tashkeela.zip
-        unzip tashkeela.zip
-        for d in `ls rababa-tashkeela-1.0/tashkeela_val/*`; do cat $d >> CA_MSA/eval.csv; done
-        for d in `ls rababa-tashkeela-1.0/tashkeela_train/*`; do cat $d >> CA_MSA/train.csv; done
-        for d in `ls rababa-tashkeela-1.0/tashkeela_test/*`; do cat $d >> CA_MSA/test.csv; done
+      - name: Prepare dataset
+        working-directory: ./python/arabic
+        run: |
+          mkdir -p data/CA_MSA
+          touch data/CA_MSA/{eval,train,test}.csv
+          cd data
+          curl -sSL https://github.com/interscript/rababa-tashkeela/archive/refs/tags/v1.0.zip -o tashkeela.zip
+          unzip tashkeela.zip
+          for d in `ls rababa-tashkeela-1.0/tashkeela_val/*`; do cat $d >> CA_MSA/eval.csv; done
+          for d in `ls rababa-tashkeela-1.0/tashkeela_train/*`; do cat $d >> CA_MSA/train.csv; done
+          for d in `ls rababa-tashkeela-1.0/tashkeela_test/*`; do cat $d >> CA_MSA/test.csv; done
 
-    - name: Try training (WIP)
-      working-directory: ./python/arabic
-      run: |
-        python train.py --model "cbhg" --config config/test_cbhg.yml
+      - name: Try training (WIP)
+        working-directory: ./python/arabic
+        run: |
+          python train.py --model "cbhg" --config config/test_cbhg.yml

.github/workflows/release.yml

@@ -9,12 +9,12 @@ jobs:
   release:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v7
 
-      - uses: actions/setup-ruby@v1
+      - uses: ruby/setup-ruby@v1
         with:
-          ruby-version: '2.7'
-          architecture: 'x64'
+          ruby-version: '3.3'
+          bundler-cache: true
 
       - run: bundle install --jobs 4 --retry 3
 
@@ -23,14 +23,10 @@ jobs:
 
       - name: Publish to rubygems.org
         env:
-          RUBYGEMS_API_KEY: ${{secrets.INTERSCRIPT_RUBYGEMS_API_KEY}}
+          RUBYGEMS_API_KEY: ${{ secrets.INTERSCRIPT_RUBYGEMS_API_KEY }}
         run: |
-          gem install gem-release
-          touch ~/.gem/credentials
-          cat > ~/.gem/credentials << EOF
-          ---
-          :rubygems_api_key: ${RUBYGEMS_API_KEY}
-          EOF
+          mkdir -p ~/.gem
+          printf -- "---\n:rubygems_api_key: %s\n" "$RUBYGEMS_API_KEY" > ~/.gem/credentials
           chmod 0600 ~/.gem/credentials
-          git status
+          gem install gem-release
           gem release

.github/workflows/ruby.yml

@@ -11,21 +11,19 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby-version: ['2.6', '2.7', '3.0', '3.1', '3.2']
+        ruby-version: ["3.3", "3.4"]
 
     steps:
-    - uses: actions/checkout@v2
+      - uses: actions/checkout@v7
 
-    - name: Set up Ruby
-      uses: ruby/setup-ruby@v1
-      with:
-        ruby-version: ${{ matrix.ruby-version }}
-        bundler-cache: true
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby-version }}
+          bundler-cache: true
 
-    - name: Run rake
-      run: |
-        bundle exec rake
+      - name: Run rake
+        run: bundle exec rake
 
-    - name: Run standardrb (use bundle exec standardrb --fix)
-      run: |
-        bundle exec standardrb
+      - name: Run standardrb
+        run: bundle exec standardrb

.gitignore

@@ -15,3 +15,32 @@ Gemfile.lock
 
 .eggs
 __pycache__
+
+# Python training artifacts — never commit
+python/log_dir/
+python/data/
+python/models/
+python/__pycache__/
+**/__pycache__/
+*.pyc
+*.pyo
+*.onnx
+*.pt
+
+# Editor
+.idea/
+.vscode/
+
+# Data caches and artifacts — fetched/cloned at build time
+data/
+models/
+runs/
+.arwiki-repo/
+.hebrew-distilled-repo/
+.hewiki-repo/
+.sefaria-repo/
+.tashkeela-repo/
+.tashkeela-full-repo/
+.venv-dictabert/
+*.bak
+python/arabic/util/constants_[A-Z]*.py