Monorepo + Task queue

This is a long time coming, but this implements a worker queue. Additionally, it reorganized the repo into npm workspace so that code can be shared between the two processes. This should improve the overall performance, reliability and scalability of breadcrum for the foreseeable future.

Debug

.dockerignore

@@ -1,16 +1,15 @@
-*.md
-.env*
-.git
-.idea
-.config
-.npm
-.github
-.DS_Store
-.gitignore
-.scannerwork
-.editorconfig
-certs
-LICENSE
-nodemon.json
-node_modules
-npm-debug.log
+**/.env*
+**/.git
+**/.idea
+**/.config
+**/.npm
+**/.github
+**/.DS_Store
+**/.gitignore
+**/.scannerwork
+**/.editorconfig
+**/certs
+**/nodemon.json
+**/node_modules
+**/npm-debug.log
+**/public

.github/dependabot.yml

@@ -3,20 +3,22 @@
 
 version: 2
 updates:
-  # Enable version updates for npm
-  - package-ecosystem: "npm"
-    # Look for `package.json` and `lock` files in the `root` directory
+  # Root
+  - package-ecosystem: "github-actions"
     directory: "/"
-    # Check the npm registry for updates every day (weekdays)
     schedule:
       interval: "daily"
-  # Enable updates to github actions
-  - package-ecosystem: "github-actions"
-    directory: "/"
+  - package-ecosystem: "npm"
+    directory: "/" # Covers all workspaces
     schedule:
       interval: "daily"
+  # Worker
   - package-ecosystem: "docker"
-    directory: "/"
+    directory: "/packages/worker"
+    schedule:
+      interval: "daily"
+  # Server
+  - package-ecosystem: "docker"
+    directory: "/packages/web"
     schedule:
       interval: "daily"
-

.github/workflows/deploy.yml

@@ -7,6 +7,14 @@ on:
         description: 'ref to deploy'
         required: true
         default: 'master'
+      deploy-server:
+        type: boolean
+        description: 'Deploy the main server'
+        default: true
+      deploy-worker:
+        type: boolean
+        description: 'Deploy the worker service'
+        default: true
 
 env:
   node_version: 21
@@ -18,36 +26,75 @@ concurrency: # prevent concurrent releases
   cancel-in-progress: true
 
 jobs:
-  flyctl_deploy:
+  flyctl_deploy_server:
     runs-on: ubuntu-latest
+    if: ${{ github.event.inputs.deploy-server }}
+    defaults:
+      run:
+        working-directory: ./services/server
     steps:
-    - uses: actions/checkout@v4
-      with:
-        ref: ${{ github.event.inputs.ref }}
-    - name: Use Node.js ${{ env.node_version }}
-      uses: actions/setup-node@v4
-      with:
-        node-version: ${{ env.node_version }}
-        # setting a registry enables the NODE_AUTH_TOKEN env variable where we can set an npm token.  REQUIRED
-        registry-url: 'https://registry.npmjs.org'
-    - name: Start Redis
-      uses: supercharge/redis-github-action@1.8.0
-      with:
-        redis-version: ${{ env.redis-version }}
-    - run: npm i
-    - run: npm run generate-default-env
-    - run: npm test
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # for api access rate limiting
-    - run: rm .env
-    - run: git reset --hard && git clean -fd
-    - uses: superfly/flyctl-actions/setup-flyctl@master
-    - run: flyctl deploy --remote-only
-      env:
-        FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
-    - name: Archive lockfile
-      uses: actions/upload-artifact@v4
-      with:
-        name: pacakge-lock.json
-        path: package-lock.json
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.inputs.ref }}
+      - name: Use Node.js ${{ env.node_version }}
+        uses: actions/setup-node@v4
+        with:
+          node-version: ${{ env.node_version }}
+          # setting a registry enables the NODE_AUTH_TOKEN env variable where we can set an npm token. REQUIRED
+          registry-url: 'https://registry.npmjs.org'
+      - name: Start Redis
+        uses: supercharge/redis-github-action@1.8.0
+        with:
+          redis-version: ${{ env.redis-version }}
+      - run: npm i --install-links --workspaces=false # Install only this package and its deps
+      - name: Archive lockfile
+        uses: actions/upload-artifact@v4
+        with:
+          name: pacakge-lock.json
+          path: package-lock.json
+      - run: npm run generate-default-env
+      - run: npm test
+      - run: rm .env
+      - run: git reset --hard && git clean -fd
+      - uses: superfly/flyctl-actions/setup-flyctl@master
+      - run: flyctl deploy --remote-only --config ./services/server/fly.toml --dockerfile ./services/server/Dockerfile
+        working-directory: ./ # Run in the root context of the directory
+        env:
+          FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
 
+  flyctl_deploy_worker:
+    needs: flyctl_deploy_server
+    runs-on: ubuntu-latest
+    if: ${{ github.event.inputs.deploy-worker && always() }}
+    defaults:
+      run:
+        working-directory: ./services/worker
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.inputs.ref }}
+      - name: Use Node.js ${{ env.node_version }}
+        uses: actions/setup-node@v4
+        with:
+          node-version: ${{ env.node_version }}
+          # setting a registry enables the NODE_AUTH_TOKEN env variable where we can set an npm token. REQUIRED
+          registry-url: 'https://registry.npmjs.org'
+      - name: Start Redis
+        uses: supercharge/redis-github-action@1.8.0
+        with:
+          redis-version: ${{ env.redis-version }}
+      - run: npm i --install-links --workspaces=false # Install only this package and its deps
+      - name: Archive lockfile
+        uses: actions/upload-artifact@v4
+        with:
+          name: pacakge-lock.json
+          path: package-lock.json
+      - run: npm run generate-default-env
+      - run: npm test
+      - run: rm .env
+      - run: git reset --hard && git clean -fd
+      - uses: superfly/flyctl-actions/setup-flyctl@master
+      - run: flyctl deploy --remote-only --config ./services/worker/fly.toml --dockerfile ./services/worker/Dockerfile
+        working-directory: ./ # Run in the root context of the directory
+        env:
+          FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}

.github/workflows/tests.yml

@@ -31,15 +31,24 @@ jobs:
       with:
         name: pacakge-lock.json
         path: package-lock.json
-    - run: npm run generate-default-env
+    - run: npm run generate-default-env --workspaces --if-present
     - run: npm test
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # for api access rate limiting
     - name: Coveralls
       uses: coverallsapp/github-action@v2
       with:
+        parallel: true
         github-token: ${{ secrets.GITHUB_TOKEN }}
-        files: .tap/report/lcov.info
+        files: pacakges/web/.tap/report/lcov.info packages/worker/.tap/report/lcov.info
+
+  coverage:
+    needs: test
+    if: ${{ always() }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Close parallel build=
+        uses: coverallsapp/github-action@v2
+        with:
+          parallel-finished: true
 
   automerge:
     needs: test

README.md

@@ -4,8 +4,8 @@
 
 Breadcrum.net internet newspaper clippings. Coming very soon.
 
-![](./web/static/screenshots/bookmark-window-dark.png#gh-dark-mode-only)
-![](./web/static/screenshots/bookmark-window-light.png#gh-light-mode-only)
+![](./services/server/web/static/screenshots/bookmark-window-dark.png#gh-dark-mode-only)
+![](./services/server/web/static/screenshots/bookmark-window-light.png#gh-light-mode-only)
 
 ## License
 

knip.json

@@ -0,0 +1,35 @@
+{
+  "ignore": [],
+  "ignoreBinaries": ["fly"],
+  "ignoreDependencies": [],
+  "ignoreWorkspaces": [],
+  "workspaces": {
+    "packages/web": {
+      "entry": [
+        "app.js",
+        "client/**/*.js",
+        "plugins/**/*.js",
+        "routes/**/{index,autohooks}.js",
+        "scripts/**/*.js",
+        "test/**/*.js"
+      ],
+      "ignoreDependencies": ["highlight.js"],
+      "ignoreBinaries": ["pg-to-dbml", "postgres://postgres"],
+      "project": ["**/*.js"]
+    },
+    "packages/worker": {
+      "entry": [
+        "app.js",
+        "plugins/**/*.js",
+        "routes/**/index.js",
+        "scripts/**/*.js",
+        "test/**/*.js"
+      ],
+      "project": ["**/*.js"]
+    },
+    "packages/*": {
+      "entry": ["**/*.js"],
+      "project": ["**/*.js"]
+    }
+  }
+}