Compare commits
9 Commits
9bd22bc5de
...
2d3e15231a
| Author | SHA1 | Date | |
|---|---|---|---|
| 2d3e15231a | |||
| a6cc5dcc4a | |||
| 958c332bf1 | |||
| 49fa4d623e | |||
| 4eefa6b337 | |||
| bef2afed66 | |||
| 5ad97d97a7 | |||
| 51b0bd4b1d | |||
| 19c770c163 |
145
flake.lock
generated
145
flake.lock
generated
@@ -130,11 +130,11 @@
|
|||||||
},
|
},
|
||||||
"den": {
|
"den": {
|
||||||
"locked": {
|
"locked": {
|
||||||
"lastModified": 1774303016,
|
"lastModified": 1774498900,
|
||||||
"narHash": "sha256-uv/Mzsh80XNtvZWaDDbo33w6M18CPLLPp+3bF1tn9Cc=",
|
"narHash": "sha256-THw/ly8KvXGQ0EI+Nhu/Eo9w8w7wtgHhAeWpteNiz/Q=",
|
||||||
"owner": "vic",
|
"owner": "vic",
|
||||||
"repo": "den",
|
"repo": "den",
|
||||||
"rev": "ece9f9ec1647e82c96e4c21bb9c7060678e21d43",
|
"rev": "eb92bbfdefd22b76fa5781e8adbeff42c4fe429e",
|
||||||
"type": "github"
|
"type": "github"
|
||||||
},
|
},
|
||||||
"original": {
|
"original": {
|
||||||
@@ -191,11 +191,11 @@
|
|||||||
"rust-analyzer-src": "rust-analyzer-src"
|
"rust-analyzer-src": "rust-analyzer-src"
|
||||||
},
|
},
|
||||||
"locked": {
|
"locked": {
|
||||||
"lastModified": 1774336807,
|
"lastModified": 1774510107,
|
||||||
"narHash": "sha256-9Wu6fqKbjHBh//Rlx0p/jFVobZwgvBlBov2W2H1kfAI=",
|
"narHash": "sha256-UDNmYHtwG73aNs48ng7x0i3BWv6lkSn5KHxkNi1ivbU=",
|
||||||
"owner": "nix-community",
|
"owner": "nix-community",
|
||||||
"repo": "fenix",
|
"repo": "fenix",
|
||||||
"rev": "2b5c85b86be24ff24e9586a56101e22f90ead9d4",
|
"rev": "620b63b95c258486d2d381bc174fa7e73adf6462",
|
||||||
"type": "github"
|
"type": "github"
|
||||||
},
|
},
|
||||||
"original": {
|
"original": {
|
||||||
@@ -414,6 +414,24 @@
|
|||||||
"type": "github"
|
"type": "github"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"flake-utils_3": {
|
||||||
|
"inputs": {
|
||||||
|
"systems": "systems_6"
|
||||||
|
},
|
||||||
|
"locked": {
|
||||||
|
"lastModified": 1731533236,
|
||||||
|
"narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
|
||||||
|
"owner": "numtide",
|
||||||
|
"repo": "flake-utils",
|
||||||
|
"rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
|
||||||
|
"type": "github"
|
||||||
|
},
|
||||||
|
"original": {
|
||||||
|
"owner": "numtide",
|
||||||
|
"repo": "flake-utils",
|
||||||
|
"type": "github"
|
||||||
|
}
|
||||||
|
},
|
||||||
"himalaya": {
|
"himalaya": {
|
||||||
"inputs": {
|
"inputs": {
|
||||||
"fenix": "fenix_2",
|
"fenix": "fenix_2",
|
||||||
@@ -441,11 +459,11 @@
|
|||||||
]
|
]
|
||||||
},
|
},
|
||||||
"locked": {
|
"locked": {
|
||||||
"lastModified": 1774293042,
|
"lastModified": 1774379316,
|
||||||
"narHash": "sha256-OEBV+Y5I4Ldu98k0KvGXRfJYh+jjE8ocCSL/dxTGs1s=",
|
"narHash": "sha256-0nGNxWDUH2Hzlj/R3Zf4FEK6fsFNB/dvewuboSRZqiI=",
|
||||||
"owner": "nix-community",
|
"owner": "nix-community",
|
||||||
"repo": "home-manager",
|
"repo": "home-manager",
|
||||||
"rev": "bc357c75e3142a31b849ba49c5299fb52c61cf59",
|
"rev": "1eb0549a1ab3fe3f5acf86668249be15fa0e64f7",
|
||||||
"type": "github"
|
"type": "github"
|
||||||
},
|
},
|
||||||
"original": {
|
"original": {
|
||||||
@@ -457,11 +475,11 @@
|
|||||||
"homebrew-cask": {
|
"homebrew-cask": {
|
||||||
"flake": false,
|
"flake": false,
|
||||||
"locked": {
|
"locked": {
|
||||||
"lastModified": 1774337162,
|
"lastModified": 1774511321,
|
||||||
"narHash": "sha256-ocx/hp4kACHjMcPKnCRk3/xZTvzm9LUQU4+OfXMZZhI=",
|
"narHash": "sha256-W8rHd4rxUT7gwxLZ0i0VQbvIrosyau+H6Fa3nzXQAhA=",
|
||||||
"owner": "homebrew",
|
"owner": "homebrew",
|
||||||
"repo": "homebrew-cask",
|
"repo": "homebrew-cask",
|
||||||
"rev": "15d634e0a2c56c85312ad79f544c847d3080ea2a",
|
"rev": "58f83e03c227201b3eaa28a1f0413eb3cd8cbd98",
|
||||||
"type": "github"
|
"type": "github"
|
||||||
},
|
},
|
||||||
"original": {
|
"original": {
|
||||||
@@ -473,11 +491,11 @@
|
|||||||
"homebrew-core": {
|
"homebrew-core": {
|
||||||
"flake": false,
|
"flake": false,
|
||||||
"locked": {
|
"locked": {
|
||||||
"lastModified": 1774336831,
|
"lastModified": 1774507765,
|
||||||
"narHash": "sha256-QHI9Py0ZW81Nik/TUzzgCd30DBPVFUz/B2Im4wz1n2Q=",
|
"narHash": "sha256-YnUOneN3De1NdT0BhpZoLr1ROfphMQQ2ogpgG8ThfdQ=",
|
||||||
"owner": "homebrew",
|
"owner": "homebrew",
|
||||||
"repo": "homebrew-core",
|
"repo": "homebrew-core",
|
||||||
"rev": "3d604bbb18b384e64fa759c7c7778a9436c7d4d5",
|
"rev": "dc21b19f04878fb04b9ed3a3fa1deb203162ece7",
|
||||||
"type": "github"
|
"type": "github"
|
||||||
},
|
},
|
||||||
"original": {
|
"original": {
|
||||||
@@ -593,11 +611,11 @@
|
|||||||
"treefmt-nix": "treefmt-nix"
|
"treefmt-nix": "treefmt-nix"
|
||||||
},
|
},
|
||||||
"locked": {
|
"locked": {
|
||||||
"lastModified": 1774333104,
|
"lastModified": 1774495900,
|
||||||
"narHash": "sha256-acWWrX21golBLxEQ+RxEow2MZ7KxFi8iCH1LUXUL4eg=",
|
"narHash": "sha256-3nR7HKulLSib37PWcWrfELuSrikFLiTqAqX2HQ9dV7g=",
|
||||||
"owner": "numtide",
|
"owner": "numtide",
|
||||||
"repo": "llm-agents.nix",
|
"repo": "llm-agents.nix",
|
||||||
"rev": "2935ee5defa6159e56cc31ee122f8caa3772c174",
|
"rev": "3e06fd5f99381f8101c8e7b5a1473154dd0095cd",
|
||||||
"type": "github"
|
"type": "github"
|
||||||
},
|
},
|
||||||
"original": {
|
"original": {
|
||||||
@@ -637,11 +655,11 @@
|
|||||||
]
|
]
|
||||||
},
|
},
|
||||||
"locked": {
|
"locked": {
|
||||||
"lastModified": 1774310655,
|
"lastModified": 1774483626,
|
||||||
"narHash": "sha256-wKLJ5XMA1x3rcJOckqp0VC6QkJ9IKJwANG37VMbYxvw=",
|
"narHash": "sha256-8VAX9GXNfv4eBj0qBEf/Rc2/E6G0SBEpuo2A5plw34I=",
|
||||||
"owner": "nix-community",
|
"owner": "nix-community",
|
||||||
"repo": "neovim-nightly-overlay",
|
"repo": "neovim-nightly-overlay",
|
||||||
"rev": "0fcef258e524f08cdcaaa163c37f14b892f6fd88",
|
"rev": "5deaa19e80e1c0695f7fa8a16e13a704fd08f96e",
|
||||||
"type": "github"
|
"type": "github"
|
||||||
},
|
},
|
||||||
"original": {
|
"original": {
|
||||||
@@ -653,11 +671,11 @@
|
|||||||
"neovim-src": {
|
"neovim-src": {
|
||||||
"flake": false,
|
"flake": false,
|
||||||
"locked": {
|
"locked": {
|
||||||
"lastModified": 1774309203,
|
"lastModified": 1774472446,
|
||||||
"narHash": "sha256-iCxl6+jjAXSLmmoYjbgO6k2/hm/a2EVxMW44oCA1GWs=",
|
"narHash": "sha256-Hp4A0llEmBvvNuw5uKOz+BA86X7TmXZ1vUK0StiMdVs=",
|
||||||
"owner": "neovim",
|
"owner": "neovim",
|
||||||
"repo": "neovim",
|
"repo": "neovim",
|
||||||
"rev": "1de1c08210076edbdc23fba86d9ffff25b54cbd8",
|
"rev": "c9e961994b16ed841be43541ef550bf3d3f043ec",
|
||||||
"type": "github"
|
"type": "github"
|
||||||
},
|
},
|
||||||
"original": {
|
"original": {
|
||||||
@@ -750,11 +768,11 @@
|
|||||||
},
|
},
|
||||||
"nixpkgs_5": {
|
"nixpkgs_5": {
|
||||||
"locked": {
|
"locked": {
|
||||||
"lastModified": 1774337345,
|
"lastModified": 1774511072,
|
||||||
"narHash": "sha256-1Bm/n42nuCHBEmSsp/mOAFLHvukAc2pyQgSDzIJ6nfY=",
|
"narHash": "sha256-I3/ioUvXgXppizwI2rfmcZU8w7PHmU2tNz27tZTOib8=",
|
||||||
"owner": "nixos",
|
"owner": "nixos",
|
||||||
"repo": "nixpkgs",
|
"repo": "nixpkgs",
|
||||||
"rev": "ebebb7072e01d84afec27c5b4581f43041c9f36d",
|
"rev": "11f32dd3f0be2df8c88fc539c627805bf3a9ff26",
|
||||||
"type": "github"
|
"type": "github"
|
||||||
},
|
},
|
||||||
"original": {
|
"original": {
|
||||||
@@ -781,6 +799,22 @@
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
"nixpkgs_7": {
|
"nixpkgs_7": {
|
||||||
|
"locked": {
|
||||||
|
"lastModified": 1769188852,
|
||||||
|
"narHash": "sha256-aBAGyMum27K7cP5OR7BMioJOF3icquJMZDDgk6ZEg1A=",
|
||||||
|
"owner": "NixOS",
|
||||||
|
"repo": "nixpkgs",
|
||||||
|
"rev": "a1bab9e494f5f4939442a57a58d0449a109593fe",
|
||||||
|
"type": "github"
|
||||||
|
},
|
||||||
|
"original": {
|
||||||
|
"owner": "NixOS",
|
||||||
|
"ref": "nixpkgs-unstable",
|
||||||
|
"repo": "nixpkgs",
|
||||||
|
"type": "github"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"nixpkgs_8": {
|
||||||
"locked": {
|
"locked": {
|
||||||
"lastModified": 1765934234,
|
"lastModified": 1765934234,
|
||||||
"narHash": "sha256-pJjWUzNnjbIAMIc5gRFUuKCDQ9S1cuh3b2hKgA7Mc4A=",
|
"narHash": "sha256-pJjWUzNnjbIAMIc5gRFUuKCDQ9S1cuh3b2hKgA7Mc4A=",
|
||||||
@@ -819,11 +853,11 @@
|
|||||||
"pi-agent-stuff": {
|
"pi-agent-stuff": {
|
||||||
"flake": false,
|
"flake": false,
|
||||||
"locked": {
|
"locked": {
|
||||||
"lastModified": 1774288055,
|
"lastModified": 1774394773,
|
||||||
"narHash": "sha256-VZFa3PCOYM3Iz8b+TXPTnQDHNpRK5wvkateuQz0lu0E=",
|
"narHash": "sha256-HguiVoKS87LEnqdUPLXv6VNDlA4zg9+QImZ3YnhlR2c=",
|
||||||
"owner": "mitsuhiko",
|
"owner": "mitsuhiko",
|
||||||
"repo": "agent-stuff",
|
"repo": "agent-stuff",
|
||||||
"rev": "7ca2deb75f4a1853bad7d93416c72838080c5e55",
|
"rev": "3bf6bd34e516af81d9c2b313b568031a785a15e2",
|
||||||
"type": "github"
|
"type": "github"
|
||||||
},
|
},
|
||||||
"original": {
|
"original": {
|
||||||
@@ -851,11 +885,11 @@
|
|||||||
"pi-harness": {
|
"pi-harness": {
|
||||||
"flake": false,
|
"flake": false,
|
||||||
"locked": {
|
"locked": {
|
||||||
"lastModified": 1774276819,
|
"lastModified": 1774378950,
|
||||||
"narHash": "sha256-BrbgnFFd4GHJhKa0MuaT2o/LvwLf4GXPbJi6j3H9AHk=",
|
"narHash": "sha256-TATerZDCk4mHCYxTf11jKPW9yPVvUDQP40RyvvoE5c8=",
|
||||||
"owner": "aliou",
|
"owner": "aliou",
|
||||||
"repo": "pi-harness",
|
"repo": "pi-harness",
|
||||||
"rev": "5bbf0cd6d6ee35d978ca32a070d9d53fbbfe2571",
|
"rev": "9a55e78b0650f63d368208d16e0d0fd7ba1e64a3",
|
||||||
"type": "github"
|
"type": "github"
|
||||||
},
|
},
|
||||||
"original": {
|
"original": {
|
||||||
@@ -913,6 +947,25 @@
|
|||||||
"type": "github"
|
"type": "github"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"qmd": {
|
||||||
|
"inputs": {
|
||||||
|
"flake-utils": "flake-utils_2",
|
||||||
|
"nixpkgs": "nixpkgs_7"
|
||||||
|
},
|
||||||
|
"locked": {
|
||||||
|
"lastModified": 1773490238,
|
||||||
|
"narHash": "sha256-13jhDU2wt4/SNR8oblwP6URr4SEVgMS6roiOpQEWQiY=",
|
||||||
|
"owner": "tobi",
|
||||||
|
"repo": "qmd",
|
||||||
|
"rev": "2b8f329d7e4419af736a50e917057f685ad41110",
|
||||||
|
"type": "github"
|
||||||
|
},
|
||||||
|
"original": {
|
||||||
|
"owner": "tobi",
|
||||||
|
"repo": "qmd",
|
||||||
|
"type": "github"
|
||||||
|
}
|
||||||
|
},
|
||||||
"root": {
|
"root": {
|
||||||
"inputs": {
|
"inputs": {
|
||||||
"code-review-nvim": "code-review-nvim",
|
"code-review-nvim": "code-review-nvim",
|
||||||
@@ -947,6 +1000,7 @@
|
|||||||
"pi-harness": "pi-harness",
|
"pi-harness": "pi-harness",
|
||||||
"pi-mcp-adapter": "pi-mcp-adapter",
|
"pi-mcp-adapter": "pi-mcp-adapter",
|
||||||
"pi-rose-pine": "pi-rose-pine",
|
"pi-rose-pine": "pi-rose-pine",
|
||||||
|
"qmd": "qmd",
|
||||||
"sops-nix": "sops-nix",
|
"sops-nix": "sops-nix",
|
||||||
"zjstatus": "zjstatus"
|
"zjstatus": "zjstatus"
|
||||||
}
|
}
|
||||||
@@ -954,11 +1008,11 @@
|
|||||||
"rust-analyzer-src": {
|
"rust-analyzer-src": {
|
||||||
"flake": false,
|
"flake": false,
|
||||||
"locked": {
|
"locked": {
|
||||||
"lastModified": 1774276450,
|
"lastModified": 1774454876,
|
||||||
"narHash": "sha256-B+2C4/9l+ggImtYBwbzEDY3v9DFcDS7hNGDUI+E460o=",
|
"narHash": "sha256-bwkM8HseUs/22x+hy6FWvJMP6q/2CKBrm4sYxz9rMY8=",
|
||||||
"owner": "rust-lang",
|
"owner": "rust-lang",
|
||||||
"repo": "rust-analyzer",
|
"repo": "rust-analyzer",
|
||||||
"rev": "da2fe7fed8960a4e351818d52bf444d7c9569da7",
|
"rev": "9253d39eab8b9c9da3c1412fc94764e01d55a02b",
|
||||||
"type": "github"
|
"type": "github"
|
||||||
},
|
},
|
||||||
"original": {
|
"original": {
|
||||||
@@ -1118,6 +1172,21 @@
|
|||||||
"type": "github"
|
"type": "github"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"systems_6": {
|
||||||
|
"locked": {
|
||||||
|
"lastModified": 1681028828,
|
||||||
|
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
|
||||||
|
"owner": "nix-systems",
|
||||||
|
"repo": "default",
|
||||||
|
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
|
||||||
|
"type": "github"
|
||||||
|
},
|
||||||
|
"original": {
|
||||||
|
"owner": "nix-systems",
|
||||||
|
"repo": "default",
|
||||||
|
"type": "github"
|
||||||
|
}
|
||||||
|
},
|
||||||
"treefmt-nix": {
|
"treefmt-nix": {
|
||||||
"inputs": {
|
"inputs": {
|
||||||
"nixpkgs": [
|
"nixpkgs": [
|
||||||
@@ -1160,8 +1229,8 @@
|
|||||||
"zjstatus": {
|
"zjstatus": {
|
||||||
"inputs": {
|
"inputs": {
|
||||||
"crane": "crane",
|
"crane": "crane",
|
||||||
"flake-utils": "flake-utils_2",
|
"flake-utils": "flake-utils_3",
|
||||||
"nixpkgs": "nixpkgs_7",
|
"nixpkgs": "nixpkgs_8",
|
||||||
"rust-overlay": "rust-overlay"
|
"rust-overlay": "rust-overlay"
|
||||||
},
|
},
|
||||||
"locked": {
|
"locked": {
|
||||||
|
|||||||
@@ -88,6 +88,7 @@
|
|||||||
url = "github:zenobi-us/pi-rose-pine";
|
url = "github:zenobi-us/pi-rose-pine";
|
||||||
flake = false;
|
flake = false;
|
||||||
};
|
};
|
||||||
|
qmd.url = "github:tobi/qmd";
|
||||||
sops-nix = {
|
sops-nix = {
|
||||||
url = "github:Mic92/sops-nix";
|
url = "github:Mic92/sops-nix";
|
||||||
inputs.nixpkgs.follows = "nixpkgs";
|
inputs.nixpkgs.follows = "nixpkgs";
|
||||||
|
|||||||
@@ -12,12 +12,16 @@
|
|||||||
- Use Nushell (`nu`) for scripting.
|
- Use Nushell (`nu`) for scripting.
|
||||||
- Do not use Python, Perl, Lua, awk, or any other scripting language. You are programatically blocked from doing so.
|
- Do not use Python, Perl, Lua, awk, or any other scripting language. You are programatically blocked from doing so.
|
||||||
|
|
||||||
|
## Workflow
|
||||||
|
|
||||||
|
- Always complete the requested work.
|
||||||
|
- If there is any ambiguity about what to do next, do NOT make a decision yourself. Stop your work and ask.
|
||||||
|
- Do not end with “If you want me to…” or “I can…”; take the next necessary step and finish the job without waiting for additional confirmation.
|
||||||
|
- Do not future-proof things. Stick to the original plan.
|
||||||
|
- Do not add fallbacks or backward compatibility unless explicitly required by the user. By default, replace the previous implementation with the new one entirely.
|
||||||
|
|
||||||
## Validation
|
## Validation
|
||||||
|
|
||||||
- Do not ignore failing tests or checks, even if they appear unrelated to your changes.
|
- Do not ignore failing tests or checks, even if they appear unrelated to your changes.
|
||||||
- After completing and validating your work, the final step is to run the project's full validation and test commands and ensure they all pass.
|
- After completing and validating your work, the final step is to run the project's full validation and test commands and ensure they all pass.
|
||||||
|
|
||||||
## Workflow
|
|
||||||
|
|
||||||
- Always complete the requested work.
|
|
||||||
- Do not end with “If you want me to…” or “I can…”; take the next necessary step and finish the job without waiting for additional confirmation.
|
|
||||||
|
|||||||
687
modules/_ai-tools/extensions/note-ingest.ts
Normal file
687
modules/_ai-tools/extensions/note-ingest.ts
Normal file
@@ -0,0 +1,687 @@
|
|||||||
|
import { readFile, writeFile, mkdir, readdir } from "node:fs/promises";
|
||||||
|
import * as fs from "node:fs";
|
||||||
|
import * as os from "node:os";
|
||||||
|
import * as path from "node:path";
|
||||||
|
import * as crypto from "node:crypto";
|
||||||
|
import { Box, Text } from "@mariozechner/pi-tui";
|
||||||
|
import type { ExtensionAPI, ExtensionContext, ExtensionCommandContext, Model } from "@mariozechner/pi-coding-agent";
|
||||||
|
import {
|
||||||
|
createAgentSession,
|
||||||
|
DefaultResourceLoader,
|
||||||
|
getAgentDir,
|
||||||
|
SessionManager,
|
||||||
|
SettingsManager,
|
||||||
|
} from "@mariozechner/pi-coding-agent";
|
||||||
|
|
||||||
|
interface IngestManifest {
|
||||||
|
version: number;
|
||||||
|
job_id: string;
|
||||||
|
note_id: string;
|
||||||
|
operation: string;
|
||||||
|
requested_at: string;
|
||||||
|
title: string;
|
||||||
|
source_relpath: string;
|
||||||
|
source_path: string;
|
||||||
|
input_path: string;
|
||||||
|
archive_path: string;
|
||||||
|
output_path: string;
|
||||||
|
transcript_path: string;
|
||||||
|
result_path: string;
|
||||||
|
session_dir: string;
|
||||||
|
source_hash: string;
|
||||||
|
last_generated_output_hash?: string | null;
|
||||||
|
force_overwrite_generated?: boolean;
|
||||||
|
source_transport?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface IngestResult {
|
||||||
|
success: boolean;
|
||||||
|
job_id: string;
|
||||||
|
note_id: string;
|
||||||
|
archive_path: string;
|
||||||
|
source_hash: string;
|
||||||
|
session_dir: string;
|
||||||
|
output_path?: string;
|
||||||
|
output_hash?: string;
|
||||||
|
conflict_path?: string;
|
||||||
|
write_mode?: "create" | "overwrite" | "force-overwrite" | "conflict";
|
||||||
|
updated_main_output?: boolean;
|
||||||
|
transcript_path?: string;
|
||||||
|
error?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface FrontmatterInfo {
|
||||||
|
values: Record<string, string>;
|
||||||
|
body: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface RenderedPage {
|
||||||
|
path: string;
|
||||||
|
image: {
|
||||||
|
type: "image";
|
||||||
|
source: {
|
||||||
|
type: "base64";
|
||||||
|
mediaType: string;
|
||||||
|
data: string;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const TRANSCRIBE_SKILL = "notability-transcribe";
|
||||||
|
const NORMALIZE_SKILL = "notability-normalize";
|
||||||
|
const STATUS_TYPE = "notability-status";
|
||||||
|
const DEFAULT_TRANSCRIBE_THINKING = "low" as const;
|
||||||
|
const DEFAULT_NORMALIZE_THINKING = "off" as const;
|
||||||
|
const PREFERRED_VISION_MODEL: [string, string] = ["openai-codex", "gpt-5.4"];
|
||||||
|
|
||||||
|
function getNotesRoot(): string {
|
||||||
|
return process.env.NOTABILITY_NOTES_DIR ?? path.join(os.homedir(), "Notes");
|
||||||
|
}
|
||||||
|
|
||||||
|
function getDataRoot(): string {
|
||||||
|
return process.env.NOTABILITY_DATA_ROOT ?? path.join(os.homedir(), ".local", "share", "notability-ingest");
|
||||||
|
}
|
||||||
|
|
||||||
|
function getRenderRoot(): string {
|
||||||
|
return process.env.NOTABILITY_RENDER_ROOT ?? path.join(getDataRoot(), "rendered-pages");
|
||||||
|
}
|
||||||
|
|
||||||
|
function getNotabilityScriptDir(): string {
|
||||||
|
return path.join(getAgentDir(), "notability");
|
||||||
|
}
|
||||||
|
|
||||||
|
function getSkillPath(skillName: string): string {
|
||||||
|
return path.join(getAgentDir(), "skills", skillName, "SKILL.md");
|
||||||
|
}
|
||||||
|
|
||||||
|
function stripFrontmatterBlock(text: string): string {
|
||||||
|
const trimmed = text.trim();
|
||||||
|
if (!trimmed.startsWith("---\n")) return trimmed;
|
||||||
|
const end = trimmed.indexOf("\n---\n", 4);
|
||||||
|
if (end === -1) return trimmed;
|
||||||
|
return trimmed.slice(end + 5).trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
function stripCodeFence(text: string): string {
|
||||||
|
const trimmed = text.trim();
|
||||||
|
const match = trimmed.match(/^```(?:markdown|md)?\n([\s\S]*?)\n```$/i);
|
||||||
|
return match ? match[1].trim() : trimmed;
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseFrontmatter(text: string): FrontmatterInfo {
|
||||||
|
const trimmed = stripCodeFence(text);
|
||||||
|
if (!trimmed.startsWith("---\n")) {
|
||||||
|
return { values: {}, body: trimmed };
|
||||||
|
}
|
||||||
|
|
||||||
|
const end = trimmed.indexOf("\n---\n", 4);
|
||||||
|
if (end === -1) {
|
||||||
|
return { values: {}, body: trimmed };
|
||||||
|
}
|
||||||
|
|
||||||
|
const block = trimmed.slice(4, end);
|
||||||
|
const body = trimmed.slice(end + 5).trim();
|
||||||
|
const values: Record<string, string> = {};
|
||||||
|
for (const line of block.split("\n")) {
|
||||||
|
const idx = line.indexOf(":");
|
||||||
|
if (idx === -1) continue;
|
||||||
|
const key = line.slice(0, idx).trim();
|
||||||
|
const value = line.slice(idx + 1).trim();
|
||||||
|
values[key] = value;
|
||||||
|
}
|
||||||
|
return { values, body };
|
||||||
|
}
|
||||||
|
|
||||||
|
function quoteYaml(value: string): string {
|
||||||
|
return JSON.stringify(value);
|
||||||
|
}
|
||||||
|
|
||||||
|
function sha256(content: string | Buffer): string {
|
||||||
|
return crypto.createHash("sha256").update(content).digest("hex");
|
||||||
|
}
|
||||||
|
|
||||||
|
async function sha256File(filePath: string): Promise<string> {
|
||||||
|
const buffer = await readFile(filePath);
|
||||||
|
return sha256(buffer);
|
||||||
|
}
|
||||||
|
|
||||||
|
function extractTitle(normalized: string, fallbackTitle: string): string {
|
||||||
|
const parsed = parseFrontmatter(normalized);
|
||||||
|
const frontmatterTitle = parsed.values.title?.replace(/^['"]|['"]$/g, "").trim();
|
||||||
|
if (frontmatterTitle) return frontmatterTitle;
|
||||||
|
const heading = parsed.body
|
||||||
|
.split("\n")
|
||||||
|
.map((line) => line.trim())
|
||||||
|
.find((line) => line.startsWith("# "));
|
||||||
|
if (heading) return heading.replace(/^#\s+/, "").trim();
|
||||||
|
return fallbackTitle;
|
||||||
|
}
|
||||||
|
|
||||||
|
function sourceFormat(filePath: string): string {
|
||||||
|
const extension = path.extname(filePath).toLowerCase();
|
||||||
|
if (extension === ".pdf") return "pdf";
|
||||||
|
if (extension === ".png") return "png";
|
||||||
|
return extension.replace(/^\./, "") || "unknown";
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildMarkdown(manifest: IngestManifest, normalized: string): string {
|
||||||
|
const parsed = parseFrontmatter(normalized);
|
||||||
|
const title = extractTitle(normalized, manifest.title);
|
||||||
|
const now = new Date().toISOString().replace(/\.\d{3}Z$/, "Z");
|
||||||
|
const created = manifest.requested_at.slice(0, 10);
|
||||||
|
const body = parsed.body.trim();
|
||||||
|
const outputBody = body.length > 0 ? body : `# ${title}\n`;
|
||||||
|
|
||||||
|
return [
|
||||||
|
"---",
|
||||||
|
`title: ${quoteYaml(title)}`,
|
||||||
|
`created: ${quoteYaml(created)}`,
|
||||||
|
`updated: ${quoteYaml(now.slice(0, 10))}`,
|
||||||
|
`source: ${quoteYaml("notability")}`,
|
||||||
|
`source_transport: ${quoteYaml(manifest.source_transport ?? "webdav")}`,
|
||||||
|
`source_relpath: ${quoteYaml(manifest.source_relpath)}`,
|
||||||
|
`note_id: ${quoteYaml(manifest.note_id)}`,
|
||||||
|
`managed_by: ${quoteYaml("notability-ingest")}`,
|
||||||
|
`source_file: ${quoteYaml(manifest.archive_path)}`,
|
||||||
|
`source_file_hash: ${quoteYaml(`sha256:${manifest.source_hash}`)}`,
|
||||||
|
`source_format: ${quoteYaml(sourceFormat(manifest.archive_path))}`,
|
||||||
|
`status: ${quoteYaml("active")}`,
|
||||||
|
"tags:",
|
||||||
|
" - handwritten",
|
||||||
|
" - notability",
|
||||||
|
"---",
|
||||||
|
"",
|
||||||
|
outputBody,
|
||||||
|
"",
|
||||||
|
].join("\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
function conflictPathFor(outputPath: string): string {
|
||||||
|
const parsed = path.parse(outputPath);
|
||||||
|
const stamp = new Date().toISOString().replace(/[:]/g, "-").replace(/\.\d{3}Z$/, "Z");
|
||||||
|
return path.join(parsed.dir, `${parsed.name}.conflict-${stamp}${parsed.ext}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function ensureParent(filePath: string): Promise<void> {
|
||||||
|
await mkdir(path.dirname(filePath), { recursive: true });
|
||||||
|
}
|
||||||
|
|
||||||
|
async function loadSkillText(skillName: string): Promise<string> {
|
||||||
|
const raw = await readFile(getSkillPath(skillName), "utf8");
|
||||||
|
return stripFrontmatterBlock(raw).trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizePathArg(arg: string): string {
|
||||||
|
return arg.startsWith("@") ? arg.slice(1) : arg;
|
||||||
|
}
|
||||||
|
|
||||||
|
function resolveModel(ctx: ExtensionContext, requireImage = false): Model {
|
||||||
|
const available = ctx.modelRegistry.getAvailable();
|
||||||
|
const matching = requireImage ? available.filter((model) => model.input.includes("image")) : available;
|
||||||
|
|
||||||
|
if (matching.length === 0) {
|
||||||
|
throw new Error(
|
||||||
|
requireImage
|
||||||
|
? "No image-capable model configured for pi note ingestion"
|
||||||
|
: "No available model configured for pi note ingestion",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (ctx.model && (!requireImage || ctx.model.input.includes("image"))) {
|
||||||
|
if (!requireImage) return ctx.model;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (requireImage) {
|
||||||
|
const [provider, id] = PREFERRED_VISION_MODEL;
|
||||||
|
const preferred = matching.find((model) => model.provider === provider && model.id === id);
|
||||||
|
if (preferred) return preferred;
|
||||||
|
|
||||||
|
const subscriptionModel = matching.find(
|
||||||
|
(model) => model.provider !== "opencode" && model.provider !== "opencode-go",
|
||||||
|
);
|
||||||
|
if (subscriptionModel) return subscriptionModel;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (ctx.model && (!requireImage || ctx.model.input.includes("image"))) {
|
||||||
|
return ctx.model;
|
||||||
|
}
|
||||||
|
|
||||||
|
return matching[0];
|
||||||
|
}
|
||||||
|
|
||||||
|
async function runSkillPrompt(
|
||||||
|
ctx: ExtensionContext,
|
||||||
|
systemPrompt: string,
|
||||||
|
prompt: string,
|
||||||
|
images: RenderedPage[] = [],
|
||||||
|
thinkingLevel: "off" | "low" = "off",
|
||||||
|
): Promise<string> {
|
||||||
|
if (images.length > 0) {
|
||||||
|
const model = resolveModel(ctx, true);
|
||||||
|
const { execFile } = await import("node:child_process");
|
||||||
|
const promptPath = path.join(os.tmpdir(), `pi-note-ingest-${crypto.randomUUID()}.md`);
|
||||||
|
await writeFile(promptPath, `${prompt}\n`);
|
||||||
|
const args = [
|
||||||
|
"45s",
|
||||||
|
"pi",
|
||||||
|
"--model",
|
||||||
|
`${model.provider}/${model.id}`,
|
||||||
|
"--thinking",
|
||||||
|
thinkingLevel,
|
||||||
|
"--no-tools",
|
||||||
|
"--no-session",
|
||||||
|
"-p",
|
||||||
|
...images.map((page) => `@${page.path}`),
|
||||||
|
`@${promptPath}`,
|
||||||
|
];
|
||||||
|
|
||||||
|
try {
|
||||||
|
const output = await new Promise<string>((resolve, reject) => {
|
||||||
|
execFile("timeout", args, { cwd: ctx.cwd, env: process.env, maxBuffer: 10 * 1024 * 1024 }, (error, stdout, stderr) => {
|
||||||
|
if ((stdout ?? "").trim().length > 0) {
|
||||||
|
resolve(stdout);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (error) {
|
||||||
|
reject(new Error(stderr || stdout || error.message));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
resolve(stdout);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
return stripCodeFence(output).trim();
|
||||||
|
} finally {
|
||||||
|
try {
|
||||||
|
fs.unlinkSync(promptPath);
|
||||||
|
} catch {
|
||||||
|
// Ignore temp file cleanup failures.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const agentDir = getAgentDir();
|
||||||
|
const settingsManager = SettingsManager.create(ctx.cwd, agentDir);
|
||||||
|
const resourceLoader = new DefaultResourceLoader({
|
||||||
|
cwd: ctx.cwd,
|
||||||
|
agentDir,
|
||||||
|
settingsManager,
|
||||||
|
noExtensions: true,
|
||||||
|
noPromptTemplates: true,
|
||||||
|
noThemes: true,
|
||||||
|
noSkills: true,
|
||||||
|
systemPromptOverride: () => systemPrompt,
|
||||||
|
appendSystemPromptOverride: () => [],
|
||||||
|
agentsFilesOverride: () => ({ agentsFiles: [] }),
|
||||||
|
});
|
||||||
|
await resourceLoader.reload();
|
||||||
|
|
||||||
|
const { session } = await createAgentSession({
|
||||||
|
model: resolveModel(ctx, images.length > 0),
|
||||||
|
thinkingLevel,
|
||||||
|
sessionManager: SessionManager.inMemory(),
|
||||||
|
modelRegistry: ctx.modelRegistry,
|
||||||
|
resourceLoader,
|
||||||
|
tools: [],
|
||||||
|
});
|
||||||
|
|
||||||
|
let output = "";
|
||||||
|
const unsubscribe = session.subscribe((event) => {
|
||||||
|
if (event.type === "message_update" && event.assistantMessageEvent.type === "text_delta") {
|
||||||
|
output += event.assistantMessageEvent.delta;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
try {
|
||||||
|
await session.prompt(prompt, {
|
||||||
|
images: images.map((page) => page.image),
|
||||||
|
});
|
||||||
|
} finally {
|
||||||
|
unsubscribe();
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!output.trim()) {
|
||||||
|
const assistantMessages = session.messages.filter((message) => message.role === "assistant");
|
||||||
|
const lastAssistant = assistantMessages.at(-1);
|
||||||
|
if (lastAssistant && Array.isArray(lastAssistant.content)) {
|
||||||
|
output = lastAssistant.content
|
||||||
|
.filter((part) => part.type === "text")
|
||||||
|
.map((part) => part.text)
|
||||||
|
.join("");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
session.dispose();
|
||||||
|
return stripCodeFence(output).trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
async function renderPdfPages(pdfPath: string, jobId: string): Promise<RenderedPage[]> {
|
||||||
|
const renderDir = path.join(getRenderRoot(), jobId);
|
||||||
|
await mkdir(renderDir, { recursive: true });
|
||||||
|
const prefix = path.join(renderDir, "page");
|
||||||
|
const args = ["-png", "-r", "200", pdfPath, prefix];
|
||||||
|
const { execFile } = await import("node:child_process");
|
||||||
|
await new Promise<void>((resolve, reject) => {
|
||||||
|
execFile("pdftoppm", args, (error) => {
|
||||||
|
if (error) reject(error);
|
||||||
|
else resolve();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
const entries = await readdir(renderDir);
|
||||||
|
const pngs = entries
|
||||||
|
.filter((entry) => entry.endsWith(".png"))
|
||||||
|
.sort((left, right) => left.localeCompare(right, undefined, { numeric: true }));
|
||||||
|
if (pngs.length === 0) {
|
||||||
|
throw new Error(`No rendered pages produced for ${pdfPath}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const pages: RenderedPage[] = [];
|
||||||
|
for (const entry of pngs) {
|
||||||
|
const pagePath = path.join(renderDir, entry);
|
||||||
|
const buffer = await readFile(pagePath);
|
||||||
|
pages.push({
|
||||||
|
path: pagePath,
|
||||||
|
image: {
|
||||||
|
type: "image",
|
||||||
|
source: {
|
||||||
|
type: "base64",
|
||||||
|
mediaType: "image/png",
|
||||||
|
data: buffer.toString("base64"),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return pages;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function loadImagePage(imagePath: string): Promise<RenderedPage> {
|
||||||
|
const extension = path.extname(imagePath).toLowerCase();
|
||||||
|
const mediaType = extension === ".png" ? "image/png" : undefined;
|
||||||
|
if (!mediaType) {
|
||||||
|
throw new Error(`Unsupported image input format for ${imagePath}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const buffer = await readFile(imagePath);
|
||||||
|
return {
|
||||||
|
path: imagePath,
|
||||||
|
image: {
|
||||||
|
type: "image",
|
||||||
|
source: {
|
||||||
|
type: "base64",
|
||||||
|
mediaType,
|
||||||
|
data: buffer.toString("base64"),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async function renderInputPages(inputPath: string, jobId: string): Promise<RenderedPage[]> {
|
||||||
|
const extension = path.extname(inputPath).toLowerCase();
|
||||||
|
if (extension === ".pdf") {
|
||||||
|
return await renderPdfPages(inputPath, jobId);
|
||||||
|
}
|
||||||
|
if (extension === ".png") {
|
||||||
|
return [await loadImagePage(inputPath)];
|
||||||
|
}
|
||||||
|
throw new Error(`Unsupported Notability input format: ${inputPath}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function findManagedOutputs(noteId: string): Promise<string[]> {
|
||||||
|
const matches: string[] = [];
|
||||||
|
const stack = [getNotesRoot()];
|
||||||
|
|
||||||
|
while (stack.length > 0) {
|
||||||
|
const currentDir = stack.pop();
|
||||||
|
if (!currentDir || !fs.existsSync(currentDir)) continue;
|
||||||
|
|
||||||
|
const entries = await readdir(currentDir, { withFileTypes: true });
|
||||||
|
for (const entry of entries) {
|
||||||
|
if (entry.name.startsWith(".")) continue;
|
||||||
|
const fullPath = path.join(currentDir, entry.name);
|
||||||
|
if (entry.isDirectory()) {
|
||||||
|
stack.push(fullPath);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (!entry.isFile() || !entry.name.endsWith(".md")) continue;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const parsed = parseFrontmatter(await readFile(fullPath, "utf8"));
|
||||||
|
const managedBy = parsed.values.managed_by?.replace(/^['"]|['"]$/g, "");
|
||||||
|
const frontmatterNoteId = parsed.values.note_id?.replace(/^['"]|['"]$/g, "");
|
||||||
|
if (managedBy === "notability-ingest" && frontmatterNoteId === noteId) {
|
||||||
|
matches.push(fullPath);
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// Ignore unreadable or malformed files while scanning the notebook.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return matches.sort();
|
||||||
|
}
|
||||||
|
|
||||||
|
async function resolveManagedOutputPath(noteId: string, configuredOutputPath: string): Promise<string> {
|
||||||
|
if (fs.existsSync(configuredOutputPath)) {
|
||||||
|
const parsed = parseFrontmatter(await readFile(configuredOutputPath, "utf8"));
|
||||||
|
const managedBy = parsed.values.managed_by?.replace(/^['"]|['"]$/g, "");
|
||||||
|
const frontmatterNoteId = parsed.values.note_id?.replace(/^['"]|['"]$/g, "");
|
||||||
|
if (managedBy === "notability-ingest" && frontmatterNoteId === noteId) {
|
||||||
|
return configuredOutputPath;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const discovered = await findManagedOutputs(noteId);
|
||||||
|
if (discovered.length === 0) return configuredOutputPath;
|
||||||
|
if (discovered.length === 1) return discovered[0];
|
||||||
|
|
||||||
|
throw new Error(
|
||||||
|
`Multiple managed note files found for ${noteId}: ${discovered.join(", ")}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function determineWriteTarget(manifest: IngestManifest, markdown: string): Promise<{
|
||||||
|
outputPath: string;
|
||||||
|
writePath: string;
|
||||||
|
writeMode: "create" | "overwrite" | "force-overwrite" | "conflict";
|
||||||
|
updatedMainOutput: boolean;
|
||||||
|
}> {
|
||||||
|
const outputPath = await resolveManagedOutputPath(manifest.note_id, manifest.output_path);
|
||||||
|
if (!fs.existsSync(outputPath)) {
|
||||||
|
return { outputPath, writePath: outputPath, writeMode: "create", updatedMainOutput: true };
|
||||||
|
}
|
||||||
|
|
||||||
|
const existing = await readFile(outputPath, "utf8");
|
||||||
|
const existingHash = sha256(existing);
|
||||||
|
const parsed = parseFrontmatter(existing);
|
||||||
|
const isManaged = parsed.values.managed_by?.replace(/^['"]|['"]$/g, "") === "notability-ingest";
|
||||||
|
const sameNoteId = parsed.values.note_id?.replace(/^['"]|['"]$/g, "") === manifest.note_id;
|
||||||
|
|
||||||
|
if (manifest.last_generated_output_hash && existingHash === manifest.last_generated_output_hash) {
|
||||||
|
return { outputPath, writePath: outputPath, writeMode: "overwrite", updatedMainOutput: true };
|
||||||
|
}
|
||||||
|
|
||||||
|
if (manifest.force_overwrite_generated && isManaged && sameNoteId) {
|
||||||
|
return { outputPath, writePath: outputPath, writeMode: "force-overwrite", updatedMainOutput: true };
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
outputPath,
|
||||||
|
writePath: conflictPathFor(outputPath),
|
||||||
|
writeMode: "conflict",
|
||||||
|
updatedMainOutput: false,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async function writeIngestResult(resultPath: string, payload: IngestResult): Promise<void> {
|
||||||
|
await ensureParent(resultPath);
|
||||||
|
await writeFile(resultPath, JSON.stringify(payload, null, 2));
|
||||||
|
}
|
||||||
|
|
||||||
|
async function ingestManifest(manifestPath: string, ctx: ExtensionContext): Promise<IngestResult> {
|
||||||
|
const manifest = JSON.parse(await readFile(manifestPath, "utf8")) as IngestManifest;
|
||||||
|
await ensureParent(manifest.transcript_path);
|
||||||
|
await ensureParent(manifest.result_path);
|
||||||
|
await mkdir(manifest.session_dir, { recursive: true });
|
||||||
|
|
||||||
|
const normalizeSkill = await loadSkillText(NORMALIZE_SKILL);
|
||||||
|
const pages = await renderInputPages(manifest.input_path, manifest.job_id);
|
||||||
|
const pageSummary = pages.map((page, index) => `- page ${index + 1}: ${page.path}`).join("\n");
|
||||||
|
const transcriptPrompt = [
|
||||||
|
"Transcribe this note into clean Markdown.",
|
||||||
|
"Read it like a human and preserve the intended reading order and visible structure.",
|
||||||
|
"Keep headings, lists, and paragraphs when they are visible.",
|
||||||
|
"Do not summarize. Do not add commentary. Return Markdown only.",
|
||||||
|
"Rendered pages:",
|
||||||
|
pageSummary,
|
||||||
|
].join("\n\n");
|
||||||
|
let transcript = await runSkillPrompt(
|
||||||
|
ctx,
|
||||||
|
"",
|
||||||
|
transcriptPrompt,
|
||||||
|
pages,
|
||||||
|
DEFAULT_TRANSCRIBE_THINKING,
|
||||||
|
);
|
||||||
|
if (!transcript.trim()) {
|
||||||
|
throw new Error("Transcription skill returned empty output");
|
||||||
|
}
|
||||||
|
await writeFile(manifest.transcript_path, `${transcript.trim()}\n`);
|
||||||
|
|
||||||
|
const normalizePrompt = [
|
||||||
|
`Note ID: ${manifest.note_id}`,
|
||||||
|
`Source path: ${manifest.source_relpath}`,
|
||||||
|
`Preferred output path: ${manifest.output_path}`,
|
||||||
|
"Normalize the following transcription into clean Markdown.",
|
||||||
|
"Restore natural prose formatting and intended reading order when the transcription contains OCR or layout artifacts.",
|
||||||
|
"If words are split across separate lines but clearly belong to the same phrase or sentence, merge them.",
|
||||||
|
"Return only Markdown. No code fences.",
|
||||||
|
"",
|
||||||
|
"<transcription>",
|
||||||
|
transcript.trim(),
|
||||||
|
"</transcription>",
|
||||||
|
].join("\n");
|
||||||
|
const normalized = await runSkillPrompt(
|
||||||
|
ctx,
|
||||||
|
normalizeSkill,
|
||||||
|
normalizePrompt,
|
||||||
|
[],
|
||||||
|
DEFAULT_NORMALIZE_THINKING,
|
||||||
|
);
|
||||||
|
if (!normalized.trim()) {
|
||||||
|
throw new Error("Normalization skill returned empty output");
|
||||||
|
}
|
||||||
|
|
||||||
|
const markdown = buildMarkdown(manifest, normalized);
|
||||||
|
const target = await determineWriteTarget(manifest, markdown);
|
||||||
|
await ensureParent(target.writePath);
|
||||||
|
await writeFile(target.writePath, markdown);
|
||||||
|
|
||||||
|
const result: IngestResult = {
|
||||||
|
success: true,
|
||||||
|
job_id: manifest.job_id,
|
||||||
|
note_id: manifest.note_id,
|
||||||
|
archive_path: manifest.archive_path,
|
||||||
|
source_hash: manifest.source_hash,
|
||||||
|
session_dir: manifest.session_dir,
|
||||||
|
output_path: target.outputPath,
|
||||||
|
output_hash: target.updatedMainOutput ? await sha256File(target.writePath) : undefined,
|
||||||
|
conflict_path: target.writeMode === "conflict" ? target.writePath : undefined,
|
||||||
|
write_mode: target.writeMode,
|
||||||
|
updated_main_output: target.updatedMainOutput,
|
||||||
|
transcript_path: manifest.transcript_path,
|
||||||
|
};
|
||||||
|
await writeIngestResult(manifest.result_path, result);
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function runScript(scriptName: string, args: string[]): Promise<string> {
|
||||||
|
const { execFile } = await import("node:child_process");
|
||||||
|
const scriptPath = path.join(getNotabilityScriptDir(), scriptName);
|
||||||
|
return await new Promise<string>((resolve, reject) => {
|
||||||
|
execFile("nu", [scriptPath, ...args], (error, stdout, stderr) => {
|
||||||
|
if (error) {
|
||||||
|
reject(new Error(stderr || stdout || error.message));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
resolve(stdout.trim());
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function splitArgs(input: string): string[] {
|
||||||
|
return input
|
||||||
|
.trim()
|
||||||
|
.split(/\s+/)
|
||||||
|
.filter((part) => part.length > 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
function postStatus(pi: ExtensionAPI, content: string): void {
|
||||||
|
pi.sendMessage({
|
||||||
|
customType: STATUS_TYPE,
|
||||||
|
content,
|
||||||
|
display: true,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export default function noteIngestExtension(pi: ExtensionAPI) {
|
||||||
|
pi.registerMessageRenderer(STATUS_TYPE, (message, _options, theme) => {
|
||||||
|
const box = new Box(1, 1, (text) => theme.bg("customMessageBg", text));
|
||||||
|
box.addChild(new Text(message.content, 0, 0));
|
||||||
|
return box;
|
||||||
|
});
|
||||||
|
|
||||||
|
pi.registerCommand("note-status", {
|
||||||
|
description: "Show Notability ingest status",
|
||||||
|
handler: async (args, _ctx) => {
|
||||||
|
const output = await runScript("status.nu", splitArgs(args));
|
||||||
|
postStatus(pi, output.length > 0 ? output : "No status output");
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
pi.registerCommand("note-reingest", {
|
||||||
|
description: "Enqueue a note for reingestion",
|
||||||
|
handler: async (args, _ctx) => {
|
||||||
|
const trimmed = args.trim();
|
||||||
|
if (!trimmed) {
|
||||||
|
postStatus(pi, "Usage: /note-reingest <note-id> [--latest-source|--latest-archive] [--force-overwrite-generated]");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const output = await runScript("reingest.nu", splitArgs(trimmed));
|
||||||
|
postStatus(pi, output.length > 0 ? output : "Reingest enqueued");
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
pi.registerCommand("note-ingest", {
|
||||||
|
description: "Ingest a queued Notability job manifest",
|
||||||
|
handler: async (args, ctx: ExtensionCommandContext) => {
|
||||||
|
const manifestPath = normalizePathArg(args.trim());
|
||||||
|
if (!manifestPath) {
|
||||||
|
throw new Error("Usage: /note-ingest <job.json>");
|
||||||
|
}
|
||||||
|
|
||||||
|
let resultPath = "";
|
||||||
|
try {
|
||||||
|
const raw = await readFile(manifestPath, "utf8");
|
||||||
|
const manifest = JSON.parse(raw) as IngestManifest;
|
||||||
|
resultPath = manifest.result_path;
|
||||||
|
const result = await ingestManifest(manifestPath, ctx);
|
||||||
|
postStatus(pi, `Ingested ${result.note_id} (${result.write_mode})`);
|
||||||
|
} catch (error) {
|
||||||
|
const message = error instanceof Error ? error.message : String(error);
|
||||||
|
if (resultPath) {
|
||||||
|
const manifest = JSON.parse(await readFile(manifestPath, "utf8")) as IngestManifest;
|
||||||
|
await writeIngestResult(resultPath, {
|
||||||
|
success: false,
|
||||||
|
job_id: manifest.job_id,
|
||||||
|
note_id: manifest.note_id,
|
||||||
|
archive_path: manifest.archive_path,
|
||||||
|
source_hash: manifest.source_hash,
|
||||||
|
session_dir: manifest.session_dir,
|
||||||
|
error: message,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
@@ -359,7 +359,7 @@ type BookmarkRef = {
|
|||||||
type ReviewTarget =
|
type ReviewTarget =
|
||||||
| { type: "workingCopy" }
|
| { type: "workingCopy" }
|
||||||
| { type: "baseBookmark"; bookmark: string; remote?: string }
|
| { type: "baseBookmark"; bookmark: string; remote?: string }
|
||||||
| { type: "change"; sha: string; title?: string }
|
| { type: "change"; changeId: string; title?: string }
|
||||||
| { type: "pullRequest"; prNumber: number; baseBookmark: string; baseRemote?: string; title: string }
|
| { type: "pullRequest"; prNumber: number; baseBookmark: string; baseRemote?: string; title: string }
|
||||||
| { type: "folder"; paths: string[] };
|
| { type: "folder"; paths: string[] };
|
||||||
|
|
||||||
@@ -377,9 +377,9 @@ const BASE_BOOKMARK_PROMPT_FALLBACK =
|
|||||||
"Review the code changes against the base bookmark '{bookmark}'. Start by finding the merge-base revision between the working copy and {bookmark}, then run `jj diff --from <merge-base> --to @` to see what changes would land on the {bookmark} bookmark. Provide prioritized, actionable findings.";
|
"Review the code changes against the base bookmark '{bookmark}'. Start by finding the merge-base revision between the working copy and {bookmark}, then run `jj diff --from <merge-base> --to @` to see what changes would land on the {bookmark} bookmark. Provide prioritized, actionable findings.";
|
||||||
|
|
||||||
const CHANGE_PROMPT_WITH_TITLE =
|
const CHANGE_PROMPT_WITH_TITLE =
|
||||||
'Review the code changes introduced by change {sha} ("{title}"). Provide prioritized, actionable findings.';
|
'Review the code changes introduced by change {changeId} ("{title}"). Provide prioritized, actionable findings.';
|
||||||
|
|
||||||
const CHANGE_PROMPT = "Review the code changes introduced by change {sha}. Provide prioritized, actionable findings.";
|
const CHANGE_PROMPT = "Review the code changes introduced by change {changeId}. Provide prioritized, actionable findings.";
|
||||||
|
|
||||||
const PULL_REQUEST_PROMPT =
|
const PULL_REQUEST_PROMPT =
|
||||||
'Review pull request #{prNumber} ("{title}") against the base bookmark \'{baseBookmark}\'. The merge-base revision for this comparison is {mergeBaseSha}. Run `jj diff --from {mergeBaseSha} --to @` to inspect the changes that would be merged. Provide prioritized, actionable findings.';
|
'Review pull request #{prNumber} ("{title}") against the base bookmark \'{baseBookmark}\'. The merge-base revision for this comparison is {mergeBaseSha}. Run `jj diff --from {mergeBaseSha} --to @` to inspect the changes that would be merged. Provide prioritized, actionable findings.';
|
||||||
@@ -748,22 +748,22 @@ async function getMergeBase(
|
|||||||
/**
|
/**
|
||||||
* Get list of recent changes
|
* Get list of recent changes
|
||||||
*/
|
*/
|
||||||
async function getRecentChanges(pi: ExtensionAPI, limit: number = 10): Promise<Array<{ sha: string; title: string }>> {
|
async function getRecentChanges(pi: ExtensionAPI, limit: number = 10): Promise<Array<{ changeId: string; title: string }>> {
|
||||||
const { stdout, code } = await pi.exec("jj", [
|
const { stdout, code } = await pi.exec("jj", [
|
||||||
"log",
|
"log",
|
||||||
"-n",
|
"-n",
|
||||||
`${limit}`,
|
`${limit}`,
|
||||||
"--no-graph",
|
"--no-graph",
|
||||||
"-T",
|
"-T",
|
||||||
'commit_id ++ "\\t" ++ description.first_line() ++ "\\n"',
|
'change_id.shortest(8) ++ "\\t" ++ description.first_line() ++ "\\n"',
|
||||||
]);
|
]);
|
||||||
if (code !== 0) return [];
|
if (code !== 0) return [];
|
||||||
|
|
||||||
return parseNonEmptyLines(stdout)
|
return parseNonEmptyLines(stdout)
|
||||||
.filter((line) => line.trim())
|
.filter((line) => line.trim())
|
||||||
.map((line) => {
|
.map((line) => {
|
||||||
const [sha, ...rest] = line.trim().split("\t");
|
const [changeId, ...rest] = line.trim().split("\t");
|
||||||
return { sha, title: rest.join(" ") };
|
return { changeId, title: rest.join(" ") };
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -979,9 +979,9 @@ async function buildReviewPrompt(
|
|||||||
|
|
||||||
case "change":
|
case "change":
|
||||||
if (target.title) {
|
if (target.title) {
|
||||||
return CHANGE_PROMPT_WITH_TITLE.replace("{sha}", target.sha).replace("{title}", target.title);
|
return CHANGE_PROMPT_WITH_TITLE.replace("{changeId}", target.changeId).replace("{title}", target.title);
|
||||||
}
|
}
|
||||||
return CHANGE_PROMPT.replace("{sha}", target.sha);
|
return CHANGE_PROMPT.replace("{changeId}", target.changeId);
|
||||||
|
|
||||||
case "pullRequest": {
|
case "pullRequest": {
|
||||||
const baseBookmarkLabel = bookmarkRefToLabel({ name: target.baseBookmark, remote: target.baseRemote });
|
const baseBookmarkLabel = bookmarkRefToLabel({ name: target.baseBookmark, remote: target.baseRemote });
|
||||||
@@ -1014,8 +1014,7 @@ function getUserFacingHint(target: ReviewTarget): string {
|
|||||||
case "baseBookmark":
|
case "baseBookmark":
|
||||||
return `changes against '${bookmarkRefToLabel({ name: target.bookmark, remote: target.remote })}'`;
|
return `changes against '${bookmarkRefToLabel({ name: target.bookmark, remote: target.remote })}'`;
|
||||||
case "change": {
|
case "change": {
|
||||||
const shortSha = target.sha.slice(0, 7);
|
return target.title ? `change ${target.changeId}: ${target.title}` : `change ${target.changeId}`;
|
||||||
return target.title ? `change ${shortSha}: ${target.title}` : `change ${shortSha}`;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
case "pullRequest": {
|
case "pullRequest": {
|
||||||
@@ -1441,12 +1440,12 @@ export default function reviewExtension(pi: ExtensionAPI) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const items: SelectItem[] = changes.map((change) => ({
|
const items: SelectItem[] = changes.map((change) => ({
|
||||||
value: change.sha,
|
value: change.changeId,
|
||||||
label: `${change.sha.slice(0, 7)} ${change.title}`,
|
label: `${change.changeId} ${change.title}`,
|
||||||
description: "",
|
description: "",
|
||||||
}));
|
}));
|
||||||
|
|
||||||
const result = await ctx.ui.custom<{ sha: string; title: string } | null>((tui, theme, keybindings, done) => {
|
const result = await ctx.ui.custom<{ changeId: string; title: string } | null>((tui, theme, keybindings, done) => {
|
||||||
const container = new Container();
|
const container = new Container();
|
||||||
container.addChild(new DynamicBorder((str) => theme.fg("accent", str)));
|
container.addChild(new DynamicBorder((str) => theme.fg("accent", str)));
|
||||||
container.addChild(new Text(theme.fg("accent", theme.bold("Select change to review"))));
|
container.addChild(new Text(theme.fg("accent", theme.bold("Select change to review"))));
|
||||||
@@ -1480,7 +1479,7 @@ export default function reviewExtension(pi: ExtensionAPI) {
|
|||||||
});
|
});
|
||||||
|
|
||||||
selectList.onSelect = (item) => {
|
selectList.onSelect = (item) => {
|
||||||
const change = changes.find((c) => c.sha === item.value);
|
const change = changes.find((c) => c.changeId === item.value);
|
||||||
if (change) {
|
if (change) {
|
||||||
done(change);
|
done(change);
|
||||||
} else {
|
} else {
|
||||||
@@ -1532,7 +1531,7 @@ export default function reviewExtension(pi: ExtensionAPI) {
|
|||||||
});
|
});
|
||||||
|
|
||||||
if (!result) return null;
|
if (!result) return null;
|
||||||
return { type: "change", sha: result.sha, title: result.title };
|
return { type: "change", changeId: result.changeId, title: result.title };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -1822,10 +1821,10 @@ export default function reviewExtension(pi: ExtensionAPI) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
case "change": {
|
case "change": {
|
||||||
const sha = parts[1];
|
const changeId = parts[1];
|
||||||
if (!sha) return { target: null, extraInstruction };
|
if (!changeId) return { target: null, extraInstruction };
|
||||||
const title = parts.slice(2).join(" ") || undefined;
|
const title = parts.slice(2).join(" ") || undefined;
|
||||||
return { target: { type: "change", sha, title }, extraInstruction };
|
return { target: { type: "change", changeId, title }, extraInstruction };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -3,15 +3,15 @@
|
|||||||
"opensrc": {
|
"opensrc": {
|
||||||
"command": "npx",
|
"command": "npx",
|
||||||
"args": ["-y", "opensrc-mcp"],
|
"args": ["-y", "opensrc-mcp"],
|
||||||
"directTools": true
|
"lifecycle": "eager"
|
||||||
},
|
},
|
||||||
"context7": {
|
"context7": {
|
||||||
"url": "https://mcp.context7.com/mcp",
|
"url": "https://mcp.context7.com/mcp",
|
||||||
"directTools": true
|
"lifecycle": "eager"
|
||||||
},
|
},
|
||||||
"grep_app": {
|
"grep_app": {
|
||||||
"url": "https://mcp.grep.app",
|
"url": "https://mcp.grep.app",
|
||||||
"directTools": true
|
"lifecycle": "eager"
|
||||||
},
|
},
|
||||||
"sentry": {
|
"sentry": {
|
||||||
"url": "https://mcp.sentry.dev/mcp",
|
"url": "https://mcp.sentry.dev/mcp",
|
||||||
|
|||||||
36
modules/_ai-tools/skills/notability-normalize/SKILL.md
Normal file
36
modules/_ai-tools/skills/notability-normalize/SKILL.md
Normal file
@@ -0,0 +1,36 @@
|
|||||||
|
---
|
||||||
|
name: notability-normalize
|
||||||
|
description: Normalizes an exact Notability transcription into clean, searchable Markdown while preserving all original content and uncertainty markers. Use after a faithful transcription pass.
|
||||||
|
---
|
||||||
|
|
||||||
|
# Notability Normalize
|
||||||
|
|
||||||
|
You are doing a **Markdown normalization** pass on a previously transcribed Notability note.
|
||||||
|
|
||||||
|
## Rules
|
||||||
|
|
||||||
|
- Do **not** summarize.
|
||||||
|
- Do **not** remove uncertainty markers such as `[unclear: ...]`.
|
||||||
|
- Preserve all substantive content from the transcription.
|
||||||
|
- Clean up only formatting and Markdown structure.
|
||||||
|
- Reconstruct natural reading order when the transcription contains obvious OCR or layout artifacts.
|
||||||
|
- Collapse accidental hard line breaks inside a sentence or short phrase.
|
||||||
|
- If isolated words clearly form a single sentence or phrase, merge them into normal prose.
|
||||||
|
- Prefer readable Markdown headings, lists, and tables.
|
||||||
|
- Keep content in the same overall order as the transcription.
|
||||||
|
- Do not invent content.
|
||||||
|
- Do not output code fences.
|
||||||
|
- Output Markdown only.
|
||||||
|
|
||||||
|
## Output
|
||||||
|
|
||||||
|
- Produce a clean Markdown document.
|
||||||
|
- Include a top-level `#` heading if the note clearly has a title.
|
||||||
|
- Use standard Markdown lists and checkboxes.
|
||||||
|
- Represent tables as Markdown tables when practical.
|
||||||
|
- Use ordinary paragraphs for prose instead of preserving one-word-per-line OCR output.
|
||||||
|
- Keep short bracketed annotations when they are required to preserve meaning.
|
||||||
|
|
||||||
|
## Important
|
||||||
|
|
||||||
|
The source PDF remains the ground truth. When in doubt, preserve ambiguity instead of cleaning it away.
|
||||||
38
modules/_ai-tools/skills/notability-transcribe/SKILL.md
Normal file
38
modules/_ai-tools/skills/notability-transcribe/SKILL.md
Normal file
@@ -0,0 +1,38 @@
|
|||||||
|
---
|
||||||
|
name: notability-transcribe
|
||||||
|
description: Faithfully transcribes handwritten or mixed handwritten/typed Notability note pages into Markdown without summarizing. Use when converting note page images or PDFs into an exact textual transcription.
|
||||||
|
---
|
||||||
|
|
||||||
|
# Notability Transcribe
|
||||||
|
|
||||||
|
You are doing a **faithful transcription** pass for handwritten Notability notes.
|
||||||
|
|
||||||
|
## Rules
|
||||||
|
|
||||||
|
- Preserve the original order of content.
|
||||||
|
- Reconstruct the intended reading order from the page layout.
|
||||||
|
- Read the page in the order a human would: top-to-bottom and left-to-right, while respecting obvious grouping.
|
||||||
|
- Do **not** summarize, explain, clean up, or reorganize beyond what is necessary to transcribe faithfully.
|
||||||
|
- Preserve headings, bullets, numbered items, checkboxes, tables, separators, callouts, and obvious layout structure.
|
||||||
|
- Do **not** preserve accidental OCR-style hard line breaks when the note is clearly continuous prose or a single phrase.
|
||||||
|
- If words are staggered on the page but clearly belong to the same sentence, combine them into normal lines.
|
||||||
|
- If text is uncertain, keep the uncertainty inline as `[unclear: ...]`.
|
||||||
|
- If a word is partially legible, include the best reading and uncertainty marker.
|
||||||
|
- If there is a drawing or diagram that cannot be represented exactly, describe it minimally in brackets, for example `[diagram: arrow from A to B]`.
|
||||||
|
- Preserve language exactly as written.
|
||||||
|
- Do not invent missing words.
|
||||||
|
- Do not output code fences.
|
||||||
|
- Output Markdown only.
|
||||||
|
|
||||||
|
## Output shape
|
||||||
|
|
||||||
|
- Use headings when headings are clearly present.
|
||||||
|
- Use `- [ ]` or `- [x]` for checkboxes when visible.
|
||||||
|
- Use bullet lists for bullet lists.
|
||||||
|
- Use normal paragraphs or single-line phrases for continuous prose instead of one word per line.
|
||||||
|
- Keep side notes in the position that best preserves reading order.
|
||||||
|
- Insert blank lines between major sections.
|
||||||
|
|
||||||
|
## Safety
|
||||||
|
|
||||||
|
If a page is partly unreadable, still transcribe everything you can and mark uncertain content with `[unclear: ...]`.
|
||||||
@@ -1,15 +1,15 @@
|
|||||||
{inputs, ...}: final: prev: let
|
{inputs, ...}: final: prev: let
|
||||||
version = "0.22.1";
|
version = "0.24.0";
|
||||||
srcs = {
|
srcs = {
|
||||||
x86_64-linux =
|
x86_64-linux =
|
||||||
prev.fetchurl {
|
prev.fetchurl {
|
||||||
url = "https://github.com/trycog/cog-cli/releases/download/v${version}/cog-linux-x86_64.tar.gz";
|
url = "https://github.com/trycog/cog-cli/releases/download/v${version}/cog-linux-x86_64.tar.gz";
|
||||||
hash = "sha256-ET+sNXisUrHShR1gxqdumegXycXcxGzJcQOdTr5005w=";
|
hash = "sha256-9Ka7rPIlWtLVxRg9yNQCNz16AE4j0zGf2TW7xBXrksM=";
|
||||||
};
|
};
|
||||||
aarch64-darwin =
|
aarch64-darwin =
|
||||||
prev.fetchurl {
|
prev.fetchurl {
|
||||||
url = "https://github.com/trycog/cog-cli/releases/download/v${version}/cog-darwin-arm64.tar.gz";
|
url = "https://github.com/trycog/cog-cli/releases/download/v${version}/cog-darwin-arm64.tar.gz";
|
||||||
hash = "sha256-jcN+DtOqr3or5C71jp7AIAz0wh73FYybCC4FRBykKO4=";
|
hash = "sha256-YNONHRmPGDhJeF+7rcWmrjqktYpi4b6bLl+M7IEFDtU=";
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
in {
|
in {
|
||||||
|
|||||||
@@ -12,7 +12,7 @@
|
|||||||
src = inputs.pi-harness;
|
src = inputs.pi-harness;
|
||||||
pnpm = prev.pnpm_10;
|
pnpm = prev.pnpm_10;
|
||||||
fetcherVersion = 1;
|
fetcherVersion = 1;
|
||||||
hash = "sha256-FgtJnmJ0/udz2A9N2DQns+a2CspMDEDk0DPUAxmCVY4=";
|
hash = "sha256-dXctaspB+efF0o88AB+1JPyWg36ZT90VeagPEBHlF20=";
|
||||||
};
|
};
|
||||||
|
|
||||||
nativeBuildInputs = [
|
nativeBuildInputs = [
|
||||||
|
|||||||
5157
modules/_overlays/qmd-package-lock.json
generated
Normal file
5157
modules/_overlays/qmd-package-lock.json
generated
Normal file
File diff suppressed because it is too large
Load Diff
43
modules/_overlays/qmd.nix
Normal file
43
modules/_overlays/qmd.nix
Normal file
@@ -0,0 +1,43 @@
|
|||||||
|
{inputs, ...}: final: prev: {
|
||||||
|
qmd =
|
||||||
|
prev.buildNpmPackage rec {
|
||||||
|
pname = "qmd";
|
||||||
|
version = "2.0.1";
|
||||||
|
src = inputs.qmd;
|
||||||
|
npmDepsHash = "sha256-ODpDkCQwkjqf9X5EfKmnCP4z4AjC6O/lS/zJKBs/46I=";
|
||||||
|
|
||||||
|
nativeBuildInputs = [
|
||||||
|
prev.makeWrapper
|
||||||
|
prev.python3
|
||||||
|
prev.pkg-config
|
||||||
|
prev.cmake
|
||||||
|
];
|
||||||
|
buildInputs = [prev.sqlite];
|
||||||
|
dontConfigure = true;
|
||||||
|
|
||||||
|
postPatch = ''
|
||||||
|
cp ${./qmd-package-lock.json} package-lock.json
|
||||||
|
'';
|
||||||
|
|
||||||
|
npmBuildScript = "build";
|
||||||
|
dontNpmPrune = true;
|
||||||
|
|
||||||
|
installPhase = ''
|
||||||
|
runHook preInstall
|
||||||
|
mkdir -p $out/lib/node_modules/qmd $out/bin
|
||||||
|
cp -r bin dist node_modules package.json package-lock.json LICENSE CHANGELOG.md $out/lib/node_modules/qmd/
|
||||||
|
makeWrapper ${prev.nodejs}/bin/node $out/bin/qmd \
|
||||||
|
--add-flags $out/lib/node_modules/qmd/dist/cli/qmd.js \
|
||||||
|
--set LD_LIBRARY_PATH ${prev.lib.makeLibraryPath [prev.sqlite]}
|
||||||
|
runHook postInstall
|
||||||
|
'';
|
||||||
|
|
||||||
|
meta = with prev.lib; {
|
||||||
|
description = "On-device search engine for markdown notes, meeting transcripts, and knowledge bases";
|
||||||
|
homepage = "https://github.com/tobi/qmd";
|
||||||
|
license = licenses.mit;
|
||||||
|
mainProgram = "qmd";
|
||||||
|
platforms = platforms.unix;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -31,13 +31,20 @@
|
|||||||
};
|
};
|
||||||
".pi/agent/extensions/no-git.ts".source = ./_ai-tools/extensions/no-git.ts;
|
".pi/agent/extensions/no-git.ts".source = ./_ai-tools/extensions/no-git.ts;
|
||||||
".pi/agent/extensions/no-scripting.ts".source = ./_ai-tools/extensions/no-scripting.ts;
|
".pi/agent/extensions/no-scripting.ts".source = ./_ai-tools/extensions/no-scripting.ts;
|
||||||
|
".pi/agent/extensions/note-ingest.ts".source = ./_ai-tools/extensions/note-ingest.ts;
|
||||||
".pi/agent/extensions/review.ts".source = ./_ai-tools/extensions/review.ts;
|
".pi/agent/extensions/review.ts".source = ./_ai-tools/extensions/review.ts;
|
||||||
".pi/agent/extensions/session-name.ts".source = ./_ai-tools/extensions/session-name.ts;
|
".pi/agent/extensions/session-name.ts".source = ./_ai-tools/extensions/session-name.ts;
|
||||||
|
".pi/agent/notability" = {
|
||||||
|
source = ./hosts/_parts/tahani/notability;
|
||||||
|
recursive = true;
|
||||||
|
};
|
||||||
".pi/agent/skills/elixir-dev" = {
|
".pi/agent/skills/elixir-dev" = {
|
||||||
source = "${inputs.pi-elixir}/skills/elixir-dev";
|
source = "${inputs.pi-elixir}/skills/elixir-dev";
|
||||||
recursive = true;
|
recursive = true;
|
||||||
};
|
};
|
||||||
".pi/agent/skills/jujutsu/SKILL.md".source = ./_ai-tools/skills/jujutsu/SKILL.md;
|
".pi/agent/skills/jujutsu/SKILL.md".source = ./_ai-tools/skills/jujutsu/SKILL.md;
|
||||||
|
".pi/agent/skills/notability-transcribe/SKILL.md".source = ./_ai-tools/skills/notability-transcribe/SKILL.md;
|
||||||
|
".pi/agent/skills/notability-normalize/SKILL.md".source = ./_ai-tools/skills/notability-normalize/SKILL.md;
|
||||||
".pi/agent/themes" = {
|
".pi/agent/themes" = {
|
||||||
source = "${inputs.pi-rose-pine}/themes";
|
source = "${inputs.pi-rose-pine}/themes";
|
||||||
recursive = true;
|
recursive = true;
|
||||||
|
|||||||
@@ -140,6 +140,11 @@
|
|||||||
|
|
||||||
homebrew = {
|
homebrew = {
|
||||||
enable = true;
|
enable = true;
|
||||||
|
onActivation = {
|
||||||
|
autoUpdate = true;
|
||||||
|
cleanup = "uninstall";
|
||||||
|
upgrade = true;
|
||||||
|
};
|
||||||
casks = [
|
casks = [
|
||||||
"1password"
|
"1password"
|
||||||
"alcove"
|
"alcove"
|
||||||
|
|||||||
@@ -74,6 +74,7 @@
|
|||||||
url = "github:nicobailon/pi-mcp-adapter/v2.2.0";
|
url = "github:nicobailon/pi-mcp-adapter/v2.2.0";
|
||||||
flake = false;
|
flake = false;
|
||||||
};
|
};
|
||||||
|
qmd.url = "github:tobi/qmd";
|
||||||
# Overlay inputs
|
# Overlay inputs
|
||||||
himalaya.url = "github:pimalaya/himalaya";
|
himalaya.url = "github:pimalaya/himalaya";
|
||||||
jj-ryu = {
|
jj-ryu = {
|
||||||
|
|||||||
128
modules/hosts/_parts/tahani/notability.nix
Normal file
128
modules/hosts/_parts/tahani/notability.nix
Normal file
@@ -0,0 +1,128 @@
|
|||||||
|
{
|
||||||
|
config,
|
||||||
|
inputs',
|
||||||
|
pkgs,
|
||||||
|
...
|
||||||
|
}: let
|
||||||
|
homeDir = "/home/cschmatzler";
|
||||||
|
notabilityScripts = ./notability;
|
||||||
|
dataRoot = "${homeDir}/.local/share/notability-ingest";
|
||||||
|
stateRoot = "${homeDir}/.local/state/notability-ingest";
|
||||||
|
notesRoot = "${homeDir}/Notes";
|
||||||
|
webdavRoot = "${dataRoot}/webdav-root";
|
||||||
|
userPackages = with pkgs; [
|
||||||
|
qmd
|
||||||
|
poppler-utils
|
||||||
|
rclone
|
||||||
|
sqlite
|
||||||
|
zk
|
||||||
|
];
|
||||||
|
commonPath = with pkgs;
|
||||||
|
[
|
||||||
|
inputs'.llm-agents.packages.pi
|
||||||
|
coreutils
|
||||||
|
inotify-tools
|
||||||
|
nushell
|
||||||
|
util-linux
|
||||||
|
]
|
||||||
|
++ userPackages;
|
||||||
|
commonEnvironment = {
|
||||||
|
HOME = homeDir;
|
||||||
|
NOTABILITY_ARCHIVE_ROOT = "${dataRoot}/archive";
|
||||||
|
NOTABILITY_DATA_ROOT = dataRoot;
|
||||||
|
NOTABILITY_DB_PATH = "${stateRoot}/db.sqlite";
|
||||||
|
NOTABILITY_NOTES_DIR = notesRoot;
|
||||||
|
NOTABILITY_RENDER_ROOT = "${dataRoot}/rendered-pages";
|
||||||
|
NOTABILITY_SESSIONS_ROOT = "${stateRoot}/sessions";
|
||||||
|
NOTABILITY_STATE_ROOT = stateRoot;
|
||||||
|
NOTABILITY_TRANSCRIPT_ROOT = "${stateRoot}/transcripts";
|
||||||
|
NOTABILITY_WEBDAV_ROOT = webdavRoot;
|
||||||
|
XDG_CONFIG_HOME = "${homeDir}/.config";
|
||||||
|
};
|
||||||
|
mkTmpDirRule = path: "d ${path} 0755 cschmatzler users -";
|
||||||
|
mkNotabilityService = {
|
||||||
|
description,
|
||||||
|
script,
|
||||||
|
after ? [],
|
||||||
|
requires ? [],
|
||||||
|
environment ? {},
|
||||||
|
}: {
|
||||||
|
inherit after description requires;
|
||||||
|
wantedBy = ["multi-user.target"];
|
||||||
|
path = commonPath;
|
||||||
|
environment = commonEnvironment // environment;
|
||||||
|
serviceConfig = {
|
||||||
|
ExecStart = "${pkgs.nushell}/bin/nu ${notabilityScripts}/${script}";
|
||||||
|
Group = "users";
|
||||||
|
Restart = "always";
|
||||||
|
RestartSec = 5;
|
||||||
|
User = "cschmatzler";
|
||||||
|
WorkingDirectory = homeDir;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
in {
|
||||||
|
sops.secrets.tahani-notability-webdav-password = {
|
||||||
|
sopsFile = ../../../../secrets/tahani-notability-webdav-password;
|
||||||
|
format = "binary";
|
||||||
|
owner = "cschmatzler";
|
||||||
|
path = "/run/secrets/tahani-notability-webdav-password";
|
||||||
|
};
|
||||||
|
|
||||||
|
home-manager.users.cschmatzler = {
|
||||||
|
home.packages = userPackages;
|
||||||
|
home.file.".config/qmd/index.yml".text = ''
|
||||||
|
collections:
|
||||||
|
notes:
|
||||||
|
path: ${notesRoot}
|
||||||
|
pattern: "**/*.md"
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
|
||||||
|
systemd.tmpfiles.rules =
|
||||||
|
builtins.map mkTmpDirRule [
|
||||||
|
notesRoot
|
||||||
|
dataRoot
|
||||||
|
webdavRoot
|
||||||
|
"${dataRoot}/archive"
|
||||||
|
"${dataRoot}/rendered-pages"
|
||||||
|
stateRoot
|
||||||
|
"${stateRoot}/jobs"
|
||||||
|
"${stateRoot}/jobs/queued"
|
||||||
|
"${stateRoot}/jobs/running"
|
||||||
|
"${stateRoot}/jobs/failed"
|
||||||
|
"${stateRoot}/jobs/done"
|
||||||
|
"${stateRoot}/jobs/results"
|
||||||
|
"${stateRoot}/sessions"
|
||||||
|
"${stateRoot}/transcripts"
|
||||||
|
];
|
||||||
|
|
||||||
|
services.caddy.virtualHosts."tahani.manticore-hippocampus.ts.net".extraConfig = ''
|
||||||
|
tls {
|
||||||
|
get_certificate tailscale
|
||||||
|
}
|
||||||
|
handle /notability* {
|
||||||
|
reverse_proxy 127.0.0.1:9980
|
||||||
|
}
|
||||||
|
'';
|
||||||
|
|
||||||
|
systemd.services.notability-webdav =
|
||||||
|
mkNotabilityService {
|
||||||
|
description = "Notability WebDAV landing zone";
|
||||||
|
script = "webdav.nu";
|
||||||
|
after = ["network.target"];
|
||||||
|
environment = {
|
||||||
|
NOTABILITY_WEBDAV_ADDR = "127.0.0.1:9980";
|
||||||
|
NOTABILITY_WEBDAV_BASEURL = "/notability";
|
||||||
|
NOTABILITY_WEBDAV_PASSWORD_FILE = config.sops.secrets.tahani-notability-webdav-password.path;
|
||||||
|
NOTABILITY_WEBDAV_USER = "notability";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
systemd.services.notability-watch =
|
||||||
|
mkNotabilityService {
|
||||||
|
description = "Watch and ingest Notability WebDAV uploads";
|
||||||
|
script = "watch.nu";
|
||||||
|
after = ["notability-webdav.service"];
|
||||||
|
requires = ["notability-webdav.service"];
|
||||||
|
};
|
||||||
|
}
|
||||||
141
modules/hosts/_parts/tahani/notability/jobs.nu
Normal file
141
modules/hosts/_parts/tahani/notability/jobs.nu
Normal file
@@ -0,0 +1,141 @@
|
|||||||
|
#!/usr/bin/env nu
|
||||||
|
|
||||||
|
use ./lib.nu *
|
||||||
|
|
||||||
|
|
||||||
|
def active-job-exists [note_id: string, source_hash: string] {
|
||||||
|
let rows = (sql-json $"
|
||||||
|
select job_id
|
||||||
|
from jobs
|
||||||
|
where note_id = (sql-quote $note_id)
|
||||||
|
and source_hash = (sql-quote $source_hash)
|
||||||
|
and status != 'done'
|
||||||
|
and status != 'failed'
|
||||||
|
limit 1;
|
||||||
|
")
|
||||||
|
not ($rows | is-empty)
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
export def archive-and-version [note_id: string, source_path: path, source_relpath: string, source_size: any, source_mtime: string, source_hash: string] {
|
||||||
|
let source_size_int = ($source_size | into int)
|
||||||
|
let archive_path = (archive-path-for $note_id $source_hash $source_relpath)
|
||||||
|
cp $source_path $archive_path
|
||||||
|
|
||||||
|
let version_id = (new-version-id)
|
||||||
|
let seen_at = (now-iso)
|
||||||
|
let version_id_q = (sql-quote $version_id)
|
||||||
|
let note_id_q = (sql-quote $note_id)
|
||||||
|
let seen_at_q = (sql-quote $seen_at)
|
||||||
|
let archive_path_q = (sql-quote $archive_path)
|
||||||
|
let source_hash_q = (sql-quote $source_hash)
|
||||||
|
let source_mtime_q = (sql-quote $source_mtime)
|
||||||
|
let source_relpath_q = (sql-quote $source_relpath)
|
||||||
|
let sql = ([
|
||||||
|
"insert into versions (version_id, note_id, seen_at, archive_path, source_hash, source_size, source_mtime, source_relpath, ingest_result, session_path) values ("
|
||||||
|
$version_id_q
|
||||||
|
", "
|
||||||
|
$note_id_q
|
||||||
|
", "
|
||||||
|
$seen_at_q
|
||||||
|
", "
|
||||||
|
$archive_path_q
|
||||||
|
", "
|
||||||
|
$source_hash_q
|
||||||
|
", "
|
||||||
|
($source_size_int | into string)
|
||||||
|
", "
|
||||||
|
$source_mtime_q
|
||||||
|
", "
|
||||||
|
$source_relpath_q
|
||||||
|
", 'pending', null);"
|
||||||
|
] | str join '')
|
||||||
|
sql-run $sql | ignore
|
||||||
|
|
||||||
|
{
|
||||||
|
version_id: $version_id
|
||||||
|
seen_at: $seen_at
|
||||||
|
archive_path: $archive_path
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
export def enqueue-job [
|
||||||
|
note: record,
|
||||||
|
operation: string,
|
||||||
|
input_path: string,
|
||||||
|
archive_path: string,
|
||||||
|
source_hash: string,
|
||||||
|
title: string,
|
||||||
|
force_overwrite_generated: bool = false,
|
||||||
|
source_transport: string = 'webdav',
|
||||||
|
] {
|
||||||
|
if (active-job-exists $note.note_id $source_hash) {
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
|
||||||
|
let job_id = (new-job-id)
|
||||||
|
let requested_at = (now-iso)
|
||||||
|
let manifest_path = (manifest-path-for $job_id 'queued')
|
||||||
|
let result_path = (result-path-for $job_id)
|
||||||
|
let transcript_path = (transcript-path-for $note.note_id $job_id)
|
||||||
|
let session_dir = ([(sessions-root) $note.note_id $job_id] | path join)
|
||||||
|
mkdir $session_dir
|
||||||
|
|
||||||
|
let manifest = {
|
||||||
|
version: 1
|
||||||
|
job_id: $job_id
|
||||||
|
note_id: $note.note_id
|
||||||
|
operation: $operation
|
||||||
|
requested_at: $requested_at
|
||||||
|
title: $title
|
||||||
|
source_relpath: $note.source_relpath
|
||||||
|
source_path: $note.source_path
|
||||||
|
input_path: $input_path
|
||||||
|
archive_path: $archive_path
|
||||||
|
output_path: $note.output_path
|
||||||
|
transcript_path: $transcript_path
|
||||||
|
result_path: $result_path
|
||||||
|
session_dir: $session_dir
|
||||||
|
source_hash: $source_hash
|
||||||
|
last_generated_output_hash: ($note.last_generated_output_hash? | default null)
|
||||||
|
force_overwrite_generated: $force_overwrite_generated
|
||||||
|
source_transport: $source_transport
|
||||||
|
}
|
||||||
|
|
||||||
|
($manifest | to json --indent 2) | save -f $manifest_path
|
||||||
|
let job_id_q = (sql-quote $job_id)
|
||||||
|
let note_id_q = (sql-quote $note.note_id)
|
||||||
|
let operation_q = (sql-quote $operation)
|
||||||
|
let requested_at_q = (sql-quote $requested_at)
|
||||||
|
let source_hash_q = (sql-quote $source_hash)
|
||||||
|
let manifest_path_q = (sql-quote $manifest_path)
|
||||||
|
let result_path_q = (sql-quote $result_path)
|
||||||
|
let sql = ([
|
||||||
|
"insert into jobs (job_id, note_id, operation, status, requested_at, source_hash, job_manifest_path, result_path) values ("
|
||||||
|
$job_id_q
|
||||||
|
", "
|
||||||
|
$note_id_q
|
||||||
|
", "
|
||||||
|
$operation_q
|
||||||
|
", 'queued', "
|
||||||
|
$requested_at_q
|
||||||
|
", "
|
||||||
|
$source_hash_q
|
||||||
|
", "
|
||||||
|
$manifest_path_q
|
||||||
|
", "
|
||||||
|
$result_path_q
|
||||||
|
");"
|
||||||
|
] | str join '')
|
||||||
|
sql-run $sql | ignore
|
||||||
|
|
||||||
|
{
|
||||||
|
job_id: $job_id
|
||||||
|
requested_at: $requested_at
|
||||||
|
manifest_path: $manifest_path
|
||||||
|
result_path: $result_path
|
||||||
|
transcript_path: $transcript_path
|
||||||
|
session_dir: $session_dir
|
||||||
|
}
|
||||||
|
}
|
||||||
433
modules/hosts/_parts/tahani/notability/lib.nu
Normal file
433
modules/hosts/_parts/tahani/notability/lib.nu
Normal file
@@ -0,0 +1,433 @@
|
|||||||
|
export def home-dir [] {
|
||||||
|
$nu.home-dir
|
||||||
|
}
|
||||||
|
|
||||||
|
export def data-root [] {
|
||||||
|
if ('NOTABILITY_DATA_ROOT' in ($env | columns)) {
|
||||||
|
$env.NOTABILITY_DATA_ROOT
|
||||||
|
} else {
|
||||||
|
[$nu.home-dir ".local" "share" "notability-ingest"] | path join
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export def state-root [] {
|
||||||
|
if ('NOTABILITY_STATE_ROOT' in ($env | columns)) {
|
||||||
|
$env.NOTABILITY_STATE_ROOT
|
||||||
|
} else {
|
||||||
|
[$nu.home-dir ".local" "state" "notability-ingest"] | path join
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export def notes-root [] {
|
||||||
|
if ('NOTABILITY_NOTES_DIR' in ($env | columns)) {
|
||||||
|
$env.NOTABILITY_NOTES_DIR
|
||||||
|
} else {
|
||||||
|
[$nu.home-dir "Notes"] | path join
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export def webdav-root [] {
|
||||||
|
if ('NOTABILITY_WEBDAV_ROOT' in ($env | columns)) {
|
||||||
|
$env.NOTABILITY_WEBDAV_ROOT
|
||||||
|
} else {
|
||||||
|
[(data-root) "webdav-root"] | path join
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export def archive-root [] {
|
||||||
|
if ('NOTABILITY_ARCHIVE_ROOT' in ($env | columns)) {
|
||||||
|
$env.NOTABILITY_ARCHIVE_ROOT
|
||||||
|
} else {
|
||||||
|
[(data-root) "archive"] | path join
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export def render-root [] {
|
||||||
|
if ('NOTABILITY_RENDER_ROOT' in ($env | columns)) {
|
||||||
|
$env.NOTABILITY_RENDER_ROOT
|
||||||
|
} else {
|
||||||
|
[(data-root) "rendered-pages"] | path join
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export def transcript-root [] {
|
||||||
|
if ('NOTABILITY_TRANSCRIPT_ROOT' in ($env | columns)) {
|
||||||
|
$env.NOTABILITY_TRANSCRIPT_ROOT
|
||||||
|
} else {
|
||||||
|
[(state-root) "transcripts"] | path join
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export def jobs-root [] {
|
||||||
|
if ('NOTABILITY_JOBS_ROOT' in ($env | columns)) {
|
||||||
|
$env.NOTABILITY_JOBS_ROOT
|
||||||
|
} else {
|
||||||
|
[(state-root) "jobs"] | path join
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export def queued-root [] {
|
||||||
|
[(jobs-root) "queued"] | path join
|
||||||
|
}
|
||||||
|
|
||||||
|
export def running-root [] {
|
||||||
|
[(jobs-root) "running"] | path join
|
||||||
|
}
|
||||||
|
|
||||||
|
export def failed-root [] {
|
||||||
|
[(jobs-root) "failed"] | path join
|
||||||
|
}
|
||||||
|
|
||||||
|
export def done-root [] {
|
||||||
|
[(jobs-root) "done"] | path join
|
||||||
|
}
|
||||||
|
|
||||||
|
export def results-root [] {
|
||||||
|
[(jobs-root) "results"] | path join
|
||||||
|
}
|
||||||
|
|
||||||
|
export def sessions-root [] {
|
||||||
|
if ('NOTABILITY_SESSIONS_ROOT' in ($env | columns)) {
|
||||||
|
$env.NOTABILITY_SESSIONS_ROOT
|
||||||
|
} else {
|
||||||
|
[(state-root) "sessions"] | path join
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export def qmd-dirty-file [] {
|
||||||
|
[(state-root) "qmd-dirty"] | path join
|
||||||
|
}
|
||||||
|
|
||||||
|
export def db-path [] {
|
||||||
|
if ('NOTABILITY_DB_PATH' in ($env | columns)) {
|
||||||
|
$env.NOTABILITY_DB_PATH
|
||||||
|
} else {
|
||||||
|
[(state-root) "db.sqlite"] | path join
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export def now-iso [] {
|
||||||
|
date now | format date "%Y-%m-%dT%H:%M:%SZ"
|
||||||
|
}
|
||||||
|
|
||||||
|
export def sql-quote [value?: any] {
|
||||||
|
if $value == null {
|
||||||
|
"NULL"
|
||||||
|
} else {
|
||||||
|
let text = ($value | into string | str replace -a "'" "''")
|
||||||
|
["'" $text "'"] | str join ''
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export def sql-run [sql: string] {
|
||||||
|
let database = (db-path)
|
||||||
|
let result = (^sqlite3 -cmd '.timeout 5000' $database $sql | complete)
|
||||||
|
if $result.exit_code != 0 {
|
||||||
|
error make {
|
||||||
|
msg: $"sqlite3 failed: ($result.stderr | str trim)"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
$result.stdout
|
||||||
|
}
|
||||||
|
|
||||||
|
export def sql-json [sql: string] {
|
||||||
|
let database = (db-path)
|
||||||
|
let result = (^sqlite3 -cmd '.timeout 5000' -json $database $sql | complete)
|
||||||
|
if $result.exit_code != 0 {
|
||||||
|
error make {
|
||||||
|
msg: $"sqlite3 failed: ($result.stderr | str trim)"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let text = ($result.stdout | str trim)
|
||||||
|
if $text == "" {
|
||||||
|
[]
|
||||||
|
} else {
|
||||||
|
$text | from json
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export def ensure-layout [] {
|
||||||
|
mkdir (data-root)
|
||||||
|
mkdir (state-root)
|
||||||
|
mkdir (notes-root)
|
||||||
|
mkdir (webdav-root)
|
||||||
|
mkdir (archive-root)
|
||||||
|
mkdir (render-root)
|
||||||
|
mkdir (transcript-root)
|
||||||
|
mkdir (jobs-root)
|
||||||
|
mkdir (queued-root)
|
||||||
|
mkdir (running-root)
|
||||||
|
mkdir (failed-root)
|
||||||
|
mkdir (done-root)
|
||||||
|
mkdir (results-root)
|
||||||
|
mkdir (sessions-root)
|
||||||
|
|
||||||
|
sql-run '
|
||||||
|
create table if not exists notes (
|
||||||
|
note_id text primary key,
|
||||||
|
source_relpath text not null unique,
|
||||||
|
title text not null,
|
||||||
|
output_path text not null,
|
||||||
|
status text not null,
|
||||||
|
first_seen_at text not null,
|
||||||
|
last_seen_at text not null,
|
||||||
|
last_processed_at text,
|
||||||
|
missing_since text,
|
||||||
|
deleted_at text,
|
||||||
|
current_source_hash text,
|
||||||
|
current_source_size integer,
|
||||||
|
current_source_mtime text,
|
||||||
|
current_archive_path text,
|
||||||
|
latest_version_id text,
|
||||||
|
last_generated_source_hash text,
|
||||||
|
last_generated_output_hash text,
|
||||||
|
conflict_path text,
|
||||||
|
last_error text
|
||||||
|
);
|
||||||
|
|
||||||
|
create table if not exists versions (
|
||||||
|
version_id text primary key,
|
||||||
|
note_id text not null,
|
||||||
|
seen_at text not null,
|
||||||
|
archive_path text not null unique,
|
||||||
|
source_hash text not null,
|
||||||
|
source_size integer not null,
|
||||||
|
source_mtime text not null,
|
||||||
|
source_relpath text not null,
|
||||||
|
ingest_result text,
|
||||||
|
session_path text,
|
||||||
|
foreign key (note_id) references notes (note_id)
|
||||||
|
);
|
||||||
|
|
||||||
|
create table if not exists jobs (
|
||||||
|
job_id text primary key,
|
||||||
|
note_id text not null,
|
||||||
|
operation text not null,
|
||||||
|
status text not null,
|
||||||
|
requested_at text not null,
|
||||||
|
started_at text,
|
||||||
|
finished_at text,
|
||||||
|
source_hash text,
|
||||||
|
job_manifest_path text not null,
|
||||||
|
result_path text not null,
|
||||||
|
error_summary text,
|
||||||
|
foreign key (note_id) references notes (note_id)
|
||||||
|
);
|
||||||
|
|
||||||
|
create table if not exists events (
|
||||||
|
id integer primary key autoincrement,
|
||||||
|
note_id text not null,
|
||||||
|
ts text not null,
|
||||||
|
kind text not null,
|
||||||
|
details text,
|
||||||
|
foreign key (note_id) references notes (note_id)
|
||||||
|
);
|
||||||
|
|
||||||
|
create index if not exists idx_jobs_status_requested_at on jobs(status, requested_at);
|
||||||
|
create index if not exists idx_versions_note_id_seen_at on versions(note_id, seen_at);
|
||||||
|
create index if not exists idx_events_note_id_ts on events(note_id, ts);
|
||||||
|
'
|
||||||
|
| ignore
|
||||||
|
}
|
||||||
|
|
||||||
|
export def log-event [note_id: string, kind: string, details?: any] {
|
||||||
|
let payload = if $details == null { null } else { $details | to json }
|
||||||
|
let note_id_q = (sql-quote $note_id)
|
||||||
|
let now_q = (sql-quote (now-iso))
|
||||||
|
let kind_q = (sql-quote $kind)
|
||||||
|
let payload_q = (sql-quote $payload)
|
||||||
|
let sql = ([
|
||||||
|
"insert into events (note_id, ts, kind, details) values ("
|
||||||
|
$note_id_q
|
||||||
|
", "
|
||||||
|
$now_q
|
||||||
|
", "
|
||||||
|
$kind_q
|
||||||
|
", "
|
||||||
|
$payload_q
|
||||||
|
");"
|
||||||
|
] | str join '')
|
||||||
|
sql-run $sql | ignore
|
||||||
|
}
|
||||||
|
|
||||||
|
export def slugify [value: string] {
|
||||||
|
let slug = (
|
||||||
|
$value
|
||||||
|
| str downcase
|
||||||
|
| str replace -r '[^a-z0-9]+' '-'
|
||||||
|
| str replace -r '^-+' ''
|
||||||
|
| str replace -r '-+$' ''
|
||||||
|
)
|
||||||
|
if $slug == '' {
|
||||||
|
'note'
|
||||||
|
} else {
|
||||||
|
$slug
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export def sha256 [file: path] {
|
||||||
|
(^sha256sum $file | lines | first | split row ' ' | first)
|
||||||
|
}
|
||||||
|
|
||||||
|
export def parse-output-frontmatter [file: path] {
|
||||||
|
if not ($file | path exists) {
|
||||||
|
{}
|
||||||
|
} else {
|
||||||
|
let content = (open --raw $file)
|
||||||
|
if not ($content | str starts-with "---\n") {
|
||||||
|
{}
|
||||||
|
} else {
|
||||||
|
let rest = ($content | str substring 4..)
|
||||||
|
let end = ($rest | str index-of "\n---\n")
|
||||||
|
if $end == null {
|
||||||
|
{}
|
||||||
|
} else {
|
||||||
|
let block = ($rest | str substring 0..($end - 1))
|
||||||
|
$block
|
||||||
|
| lines
|
||||||
|
| where ($it | str contains ':')
|
||||||
|
| reduce --fold {} {|line, acc|
|
||||||
|
let idx = ($line | str index-of ':')
|
||||||
|
if $idx == null {
|
||||||
|
$acc
|
||||||
|
} else {
|
||||||
|
let key = ($line | str substring 0..($idx - 1) | str trim)
|
||||||
|
let value = ($line | str substring ($idx + 1).. | str trim)
|
||||||
|
$acc | upsert $key $value
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export def zk-generated-note-path [title: string] {
|
||||||
|
let root = (notes-root)
|
||||||
|
let effective_title = if ($title | str trim) == '' {
|
||||||
|
'Imported note'
|
||||||
|
} else {
|
||||||
|
$title
|
||||||
|
}
|
||||||
|
let result = (
|
||||||
|
^zk --notebook-dir $root --working-dir $root new $root --no-input --title $effective_title --print-path --dry-run
|
||||||
|
| complete
|
||||||
|
)
|
||||||
|
|
||||||
|
if $result.exit_code != 0 {
|
||||||
|
error make {
|
||||||
|
msg: $"zk failed to generate a note path: ($result.stderr | str trim)"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let path_text = ($result.stderr | str trim)
|
||||||
|
if $path_text == '' {
|
||||||
|
error make {
|
||||||
|
msg: 'zk did not return a generated note path'
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
$path_text
|
||||||
|
| lines
|
||||||
|
| last
|
||||||
|
| str trim
|
||||||
|
}
|
||||||
|
|
||||||
|
export def new-note-id [] {
|
||||||
|
let suffix = (random uuid | str replace -a '-' '')
|
||||||
|
$"ntl_($suffix)"
|
||||||
|
}
|
||||||
|
|
||||||
|
export def new-job-id [] {
|
||||||
|
let suffix = (random uuid | str replace -a '-' '')
|
||||||
|
$"job_($suffix)"
|
||||||
|
}
|
||||||
|
|
||||||
|
export def new-version-id [] {
|
||||||
|
let suffix = (random uuid | str replace -a '-' '')
|
||||||
|
$"ver_($suffix)"
|
||||||
|
}
|
||||||
|
|
||||||
|
export def archive-path-for [note_id: string, source_hash: string, source_relpath: string] {
|
||||||
|
let stamp = (date now | format date "%Y-%m-%dT%H-%M-%SZ")
|
||||||
|
let short = ($source_hash | str substring 0..11)
|
||||||
|
let directory = [(archive-root) $note_id] | path join
|
||||||
|
let parsed = ($source_relpath | path parse)
|
||||||
|
let extension = if (($parsed.extension? | default '') | str trim) == '' {
|
||||||
|
'bin'
|
||||||
|
} else {
|
||||||
|
($parsed.extension | str downcase)
|
||||||
|
}
|
||||||
|
mkdir $directory
|
||||||
|
[$directory $"($stamp)-($short).($extension)"] | path join
|
||||||
|
}
|
||||||
|
|
||||||
|
export def transcript-path-for [note_id: string, job_id: string] {
|
||||||
|
let directory = [(transcript-root) $note_id] | path join
|
||||||
|
mkdir $directory
|
||||||
|
[$directory $"($job_id).md"] | path join
|
||||||
|
}
|
||||||
|
|
||||||
|
export def result-path-for [job_id: string] {
|
||||||
|
[(results-root) $"($job_id).json"] | path join
|
||||||
|
}
|
||||||
|
|
||||||
|
export def manifest-path-for [job_id: string, status: string] {
|
||||||
|
let root = match $status {
|
||||||
|
'queued' => (queued-root)
|
||||||
|
'running' => (running-root)
|
||||||
|
'failed' => (failed-root)
|
||||||
|
'done' => (done-root)
|
||||||
|
_ => (queued-root)
|
||||||
|
}
|
||||||
|
[$root $"($job_id).json"] | path join
|
||||||
|
}
|
||||||
|
|
||||||
|
export def note-output-path [title: string] {
|
||||||
|
zk-generated-note-path $title
|
||||||
|
}
|
||||||
|
|
||||||
|
export def is-supported-source-path [path: string] {
|
||||||
|
let lower = ($path | str downcase)
|
||||||
|
(($lower | str ends-with '.pdf') or ($lower | str ends-with '.png'))
|
||||||
|
}
|
||||||
|
|
||||||
|
export def is-ignored-path [relpath: string] {
|
||||||
|
let lower = ($relpath | str downcase)
|
||||||
|
let hidden = (($lower | str contains '/.') or ($lower | str starts-with '.'))
|
||||||
|
let temp = (($lower | str contains '/~') or ($lower | str ends-with '.tmp') or ($lower | str ends-with '.part'))
|
||||||
|
let conflict = ($lower | str contains '.sync-conflict')
|
||||||
|
($hidden or $temp or $conflict)
|
||||||
|
}
|
||||||
|
|
||||||
|
export def scan-source-files [] {
|
||||||
|
let root = (webdav-root)
|
||||||
|
if not ($root | path exists) {
|
||||||
|
[]
|
||||||
|
} else {
|
||||||
|
let files = ([
|
||||||
|
(glob $"($root)/**/*.pdf")
|
||||||
|
(glob $"($root)/**/*.PDF")
|
||||||
|
(glob $"($root)/**/*.png")
|
||||||
|
(glob $"($root)/**/*.PNG")
|
||||||
|
] | flatten)
|
||||||
|
$files
|
||||||
|
| sort
|
||||||
|
| uniq
|
||||||
|
| each {|file|
|
||||||
|
let relpath = ($file | path relative-to $root)
|
||||||
|
if ((is-ignored-path $relpath) or not (is-supported-source-path $file)) {
|
||||||
|
null
|
||||||
|
} else {
|
||||||
|
let stat = (ls -l $file | first)
|
||||||
|
{
|
||||||
|
source_path: $file
|
||||||
|
source_relpath: $relpath
|
||||||
|
source_size: $stat.size
|
||||||
|
source_mtime: ($stat.modified | format date "%Y-%m-%dT%H:%M:%SZ")
|
||||||
|
title: (($relpath | path parse).stem)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
| where $it != null
|
||||||
|
}
|
||||||
|
}
|
||||||
387
modules/hosts/_parts/tahani/notability/reconcile.nu
Normal file
387
modules/hosts/_parts/tahani/notability/reconcile.nu
Normal file
@@ -0,0 +1,387 @@
|
|||||||
|
#!/usr/bin/env nu
|
||||||
|
|
||||||
|
use ./lib.nu *
|
||||||
|
use ./jobs.nu [archive-and-version, enqueue-job]
|
||||||
|
|
||||||
|
const settle_window = 45sec
|
||||||
|
const delete_grace = 15min
|
||||||
|
|
||||||
|
|
||||||
|
def settle-remaining [source_mtime: string] {
|
||||||
|
let modified = ($source_mtime | into datetime)
|
||||||
|
let age = ((date now) - $modified)
|
||||||
|
if $age >= $settle_window {
|
||||||
|
0sec
|
||||||
|
} else {
|
||||||
|
$settle_window - $age
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def is-settled [source_mtime: string] {
|
||||||
|
let modified = ($source_mtime | into datetime)
|
||||||
|
((date now) - $modified) >= $settle_window
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def log-job-enqueued [note_id: string, job_id: string, operation: string, source_hash: string, archive_path: string] {
|
||||||
|
log-event $note_id 'job-enqueued' {
|
||||||
|
job_id: $job_id
|
||||||
|
operation: $operation
|
||||||
|
source_hash: $source_hash
|
||||||
|
archive_path: $archive_path
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def find-rename-candidate [source_hash: string] {
|
||||||
|
sql-json $"
|
||||||
|
select *
|
||||||
|
from notes
|
||||||
|
where current_source_hash = (sql-quote $source_hash)
|
||||||
|
and status != 'active'
|
||||||
|
and status != 'failed'
|
||||||
|
and status != 'conflict'
|
||||||
|
order by last_seen_at desc
|
||||||
|
limit 1;
|
||||||
|
"
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def touch-note [note_id: string, source_size: any, source_mtime: string, status: string = 'active'] {
|
||||||
|
let source_size_int = ($source_size | into int)
|
||||||
|
let now_q = (sql-quote (now-iso))
|
||||||
|
let source_mtime_q = (sql-quote $source_mtime)
|
||||||
|
let status_q = (sql-quote $status)
|
||||||
|
let note_id_q = (sql-quote $note_id)
|
||||||
|
sql-run $"
|
||||||
|
update notes
|
||||||
|
set last_seen_at = ($now_q),
|
||||||
|
current_source_size = ($source_size_int),
|
||||||
|
current_source_mtime = ($source_mtime_q),
|
||||||
|
status = ($status_q)
|
||||||
|
where note_id = ($note_id_q);
|
||||||
|
"
|
||||||
|
| ignore
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def process-existing [note: record, source: record] {
|
||||||
|
let title = $source.title
|
||||||
|
let note_id = ($note | get note_id)
|
||||||
|
let note_status = ($note | get status)
|
||||||
|
let source_size_int = ($source.source_size | into int)
|
||||||
|
if not (is-settled $source.source_mtime) {
|
||||||
|
touch-note $note_id $source_size_int $source.source_mtime $note_status
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
let previous_size = ($note.current_source_size? | default (-1))
|
||||||
|
let previous_mtime = ($note.current_source_mtime? | default '')
|
||||||
|
let size_changed = ($previous_size != $source_size_int)
|
||||||
|
let mtime_changed = ($previous_mtime != $source.source_mtime)
|
||||||
|
let needs_ingest = (($note.last_generated_source_hash? | default '') != ($note.current_source_hash? | default ''))
|
||||||
|
let hash_needed = ($note.current_source_hash? | default null) == null or $size_changed or $mtime_changed or ($note_status != 'active') or $needs_ingest
|
||||||
|
|
||||||
|
if not $hash_needed {
|
||||||
|
let now_q = (sql-quote (now-iso))
|
||||||
|
let title_q = (sql-quote $title)
|
||||||
|
let note_id_q = (sql-quote $note_id)
|
||||||
|
sql-run $"
|
||||||
|
update notes
|
||||||
|
set last_seen_at = ($now_q),
|
||||||
|
status = 'active',
|
||||||
|
title = ($title_q),
|
||||||
|
missing_since = null,
|
||||||
|
deleted_at = null
|
||||||
|
where note_id = ($note_id_q);
|
||||||
|
"
|
||||||
|
| ignore
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
let source_hash = (sha256 $source.source_path)
|
||||||
|
if ($source_hash == ($note.current_source_hash? | default '')) {
|
||||||
|
let now_q = (sql-quote (now-iso))
|
||||||
|
let title_q = (sql-quote $title)
|
||||||
|
let source_mtime_q = (sql-quote $source.source_mtime)
|
||||||
|
let note_id_q = (sql-quote $note_id)
|
||||||
|
let next_status = if $note_status == 'failed' { 'failed' } else { 'active' }
|
||||||
|
sql-run $"
|
||||||
|
update notes
|
||||||
|
set last_seen_at = ($now_q),
|
||||||
|
title = ($title_q),
|
||||||
|
status = (sql-quote $next_status),
|
||||||
|
missing_since = null,
|
||||||
|
deleted_at = null,
|
||||||
|
current_source_size = ($source_size_int),
|
||||||
|
current_source_mtime = ($source_mtime_q)
|
||||||
|
where note_id = ($note_id_q);
|
||||||
|
"
|
||||||
|
| ignore
|
||||||
|
|
||||||
|
let should_enqueue = ($note_status == 'failed' or (($note.last_generated_source_hash? | default '') != $source_hash))
|
||||||
|
if not $should_enqueue {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
let archive_path = if (($note.current_archive_path? | default '') | str trim) == '' {
|
||||||
|
let version = (archive-and-version $note_id $source.source_path $source.source_relpath $source_size_int $source.source_mtime $source_hash)
|
||||||
|
let archive_path_q = (sql-quote $version.archive_path)
|
||||||
|
let version_id_q = (sql-quote $version.version_id)
|
||||||
|
sql-run $"
|
||||||
|
update notes
|
||||||
|
set current_archive_path = ($archive_path_q),
|
||||||
|
latest_version_id = ($version_id_q)
|
||||||
|
where note_id = ($note_id_q);
|
||||||
|
"
|
||||||
|
| ignore
|
||||||
|
$version.archive_path
|
||||||
|
} else {
|
||||||
|
$note.current_archive_path
|
||||||
|
}
|
||||||
|
|
||||||
|
let runtime_note = ($note | upsert source_path $source.source_path | upsert source_relpath $source.source_relpath | upsert output_path $note.output_path | upsert last_generated_output_hash ($note.last_generated_output_hash? | default null))
|
||||||
|
let retry_job = (enqueue-job $runtime_note 'upsert' $archive_path $archive_path $source_hash $title)
|
||||||
|
if $retry_job != null {
|
||||||
|
log-job-enqueued $note_id $retry_job.job_id 'upsert' $source_hash $archive_path
|
||||||
|
let reason = if $note_status == 'failed' {
|
||||||
|
'retry-failed-note'
|
||||||
|
} else {
|
||||||
|
'missing-generated-output'
|
||||||
|
}
|
||||||
|
log-event $note_id 'job-reenqueued' {
|
||||||
|
job_id: $retry_job.job_id
|
||||||
|
reason: $reason
|
||||||
|
source_hash: $source_hash
|
||||||
|
archive_path: $archive_path
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
let version = (archive-and-version $note_id $source.source_path $source.source_relpath $source_size_int $source.source_mtime $source_hash)
|
||||||
|
let now_q = (sql-quote (now-iso))
|
||||||
|
let title_q = (sql-quote $title)
|
||||||
|
let source_hash_q = (sql-quote $source_hash)
|
||||||
|
let source_mtime_q = (sql-quote $source.source_mtime)
|
||||||
|
let archive_path_q = (sql-quote $version.archive_path)
|
||||||
|
let version_id_q = (sql-quote $version.version_id)
|
||||||
|
let note_id_q = (sql-quote $note_id)
|
||||||
|
sql-run $"
|
||||||
|
update notes
|
||||||
|
set last_seen_at = ($now_q),
|
||||||
|
title = ($title_q),
|
||||||
|
status = 'active',
|
||||||
|
missing_since = null,
|
||||||
|
deleted_at = null,
|
||||||
|
current_source_hash = ($source_hash_q),
|
||||||
|
current_source_size = ($source_size_int),
|
||||||
|
current_source_mtime = ($source_mtime_q),
|
||||||
|
current_archive_path = ($archive_path_q),
|
||||||
|
latest_version_id = ($version_id_q),
|
||||||
|
last_error = null
|
||||||
|
where note_id = ($note_id_q);
|
||||||
|
"
|
||||||
|
| ignore
|
||||||
|
|
||||||
|
let runtime_note = ($note | upsert source_path $source.source_path | upsert source_relpath $source.source_relpath | upsert output_path $note.output_path | upsert last_generated_output_hash ($note.last_generated_output_hash? | default null))
|
||||||
|
let job = (enqueue-job $runtime_note 'upsert' $version.archive_path $version.archive_path $source_hash $title)
|
||||||
|
if $job != null {
|
||||||
|
log-job-enqueued $note_id $job.job_id 'upsert' $source_hash $version.archive_path
|
||||||
|
}
|
||||||
|
|
||||||
|
log-event $note_id 'source-updated' {
|
||||||
|
source_relpath: $source.source_relpath
|
||||||
|
source_hash: $source_hash
|
||||||
|
archive_path: $version.archive_path
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def process-new [source: record] {
|
||||||
|
if not (is-settled $source.source_mtime) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
let source_hash = (sha256 $source.source_path)
|
||||||
|
let source_size_int = ($source.source_size | into int)
|
||||||
|
let rename_candidates = (find-rename-candidate $source_hash)
|
||||||
|
if not ($rename_candidates | is-empty) {
|
||||||
|
let rename_candidate = ($rename_candidates | first)
|
||||||
|
let source_relpath_q = (sql-quote $source.source_relpath)
|
||||||
|
let title_q = (sql-quote $source.title)
|
||||||
|
let now_q = (sql-quote (now-iso))
|
||||||
|
let source_mtime_q = (sql-quote $source.source_mtime)
|
||||||
|
let note_id_q = (sql-quote $rename_candidate.note_id)
|
||||||
|
sql-run $"
|
||||||
|
update notes
|
||||||
|
set source_relpath = ($source_relpath_q),
|
||||||
|
title = ($title_q),
|
||||||
|
last_seen_at = ($now_q),
|
||||||
|
status = 'active',
|
||||||
|
missing_since = null,
|
||||||
|
deleted_at = null,
|
||||||
|
current_source_size = ($source_size_int),
|
||||||
|
current_source_mtime = ($source_mtime_q)
|
||||||
|
where note_id = ($note_id_q);
|
||||||
|
"
|
||||||
|
| ignore
|
||||||
|
log-event $rename_candidate.note_id 'source-renamed' {
|
||||||
|
from: $rename_candidate.source_relpath
|
||||||
|
to: $source.source_relpath
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
let note_id = (new-note-id)
|
||||||
|
let first_seen_at = (now-iso)
|
||||||
|
let output_path = (note-output-path $source.title)
|
||||||
|
let version = (archive-and-version $note_id $source.source_path $source.source_relpath $source_size_int $source.source_mtime $source_hash)
|
||||||
|
let note_id_q = (sql-quote $note_id)
|
||||||
|
let source_relpath_q = (sql-quote $source.source_relpath)
|
||||||
|
let title_q = (sql-quote $source.title)
|
||||||
|
let output_path_q = (sql-quote $output_path)
|
||||||
|
let first_seen_q = (sql-quote $first_seen_at)
|
||||||
|
let source_hash_q = (sql-quote $source_hash)
|
||||||
|
let source_mtime_q = (sql-quote $source.source_mtime)
|
||||||
|
let archive_path_q = (sql-quote $version.archive_path)
|
||||||
|
let version_id_q = (sql-quote $version.version_id)
|
||||||
|
let sql = ([
|
||||||
|
"insert into notes (note_id, source_relpath, title, output_path, status, first_seen_at, last_seen_at, current_source_hash, current_source_size, current_source_mtime, current_archive_path, latest_version_id) values ("
|
||||||
|
$note_id_q
|
||||||
|
", "
|
||||||
|
$source_relpath_q
|
||||||
|
", "
|
||||||
|
$title_q
|
||||||
|
", "
|
||||||
|
$output_path_q
|
||||||
|
", 'active', "
|
||||||
|
$first_seen_q
|
||||||
|
", "
|
||||||
|
$first_seen_q
|
||||||
|
", "
|
||||||
|
$source_hash_q
|
||||||
|
", "
|
||||||
|
($source_size_int | into string)
|
||||||
|
", "
|
||||||
|
$source_mtime_q
|
||||||
|
", "
|
||||||
|
$archive_path_q
|
||||||
|
", "
|
||||||
|
$version_id_q
|
||||||
|
");"
|
||||||
|
] | str join '')
|
||||||
|
sql-run $sql | ignore
|
||||||
|
|
||||||
|
let note = {
|
||||||
|
note_id: $note_id
|
||||||
|
source_relpath: $source.source_relpath
|
||||||
|
source_path: $source.source_path
|
||||||
|
output_path: $output_path
|
||||||
|
last_generated_output_hash: null
|
||||||
|
}
|
||||||
|
let job = (enqueue-job $note 'upsert' $version.archive_path $version.archive_path $source_hash $source.title)
|
||||||
|
if $job != null {
|
||||||
|
log-job-enqueued $note_id $job.job_id 'upsert' $source_hash $version.archive_path
|
||||||
|
}
|
||||||
|
|
||||||
|
log-event $note_id 'source-discovered' {
|
||||||
|
source_relpath: $source.source_relpath
|
||||||
|
source_hash: $source_hash
|
||||||
|
archive_path: $version.archive_path
|
||||||
|
output_path: $output_path
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def mark-missing [seen_relpaths: list<string>] {
|
||||||
|
let notes = (sql-json 'select note_id, source_relpath, status, missing_since from notes;')
|
||||||
|
for note in $notes {
|
||||||
|
if ($seen_relpaths | any {|rel| $rel == $note.source_relpath }) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if $note.status == 'active' {
|
||||||
|
let missing_since = (now-iso)
|
||||||
|
let missing_since_q = (sql-quote $missing_since)
|
||||||
|
let note_id_q = (sql-quote $note.note_id)
|
||||||
|
sql-run $"
|
||||||
|
update notes
|
||||||
|
set status = 'source_missing',
|
||||||
|
missing_since = ($missing_since_q)
|
||||||
|
where note_id = ($note_id_q);
|
||||||
|
"
|
||||||
|
| ignore
|
||||||
|
log-event $note.note_id 'source-missing' {
|
||||||
|
source_relpath: $note.source_relpath
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if $note.status == 'source_missing' and ($note.missing_since? | default null) != null {
|
||||||
|
let missing_since = ($note.missing_since | into datetime)
|
||||||
|
if ((date now) - $missing_since) >= $delete_grace {
|
||||||
|
let deleted_at = (now-iso)
|
||||||
|
let deleted_at_q = (sql-quote $deleted_at)
|
||||||
|
let note_id_q = (sql-quote $note.note_id)
|
||||||
|
sql-run $"
|
||||||
|
update notes
|
||||||
|
set status = 'source_deleted',
|
||||||
|
deleted_at = ($deleted_at_q)
|
||||||
|
where note_id = ($note_id_q);
|
||||||
|
"
|
||||||
|
| ignore
|
||||||
|
log-event $note.note_id 'source-deleted' {
|
||||||
|
source_relpath: $note.source_relpath
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
export def reconcile-run [] {
|
||||||
|
ensure-layout
|
||||||
|
mut sources = (scan-source-files)
|
||||||
|
|
||||||
|
let unsettled = (
|
||||||
|
$sources
|
||||||
|
| each {|source|
|
||||||
|
{
|
||||||
|
source_path: $source.source_path
|
||||||
|
remaining: (settle-remaining $source.source_mtime)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
| where remaining > 0sec
|
||||||
|
)
|
||||||
|
|
||||||
|
if not ($unsettled | is-empty) {
|
||||||
|
let max_remaining = ($unsettled | get remaining | math max)
|
||||||
|
print $"Waiting ($max_remaining) for recent Notability uploads to settle"
|
||||||
|
sleep ($max_remaining + 2sec)
|
||||||
|
$sources = (scan-source-files)
|
||||||
|
}
|
||||||
|
|
||||||
|
for source in $sources {
|
||||||
|
let existing_rows = (sql-json $"
|
||||||
|
select *
|
||||||
|
from notes
|
||||||
|
where source_relpath = (sql-quote $source.source_relpath)
|
||||||
|
limit 1;
|
||||||
|
")
|
||||||
|
if (($existing_rows | length) == 0) {
|
||||||
|
process-new $source
|
||||||
|
} else {
|
||||||
|
let existing = ($existing_rows | first)
|
||||||
|
process-existing ($existing | upsert source_path $source.source_path) $source
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
mark-missing ($sources | get source_relpath)
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def main [] {
|
||||||
|
reconcile-run
|
||||||
|
}
|
||||||
148
modules/hosts/_parts/tahani/notability/reingest.nu
Normal file
148
modules/hosts/_parts/tahani/notability/reingest.nu
Normal file
@@ -0,0 +1,148 @@
|
|||||||
|
#!/usr/bin/env nu
|
||||||
|
|
||||||
|
use ./lib.nu *
|
||||||
|
use ./jobs.nu [archive-and-version, enqueue-job]
|
||||||
|
use ./worker.nu [worker-run]
|
||||||
|
|
||||||
|
|
||||||
|
def latest-version [note_id: string] {
|
||||||
|
sql-json $"
|
||||||
|
select *
|
||||||
|
from versions
|
||||||
|
where note_id = (sql-quote $note_id)
|
||||||
|
order by seen_at desc
|
||||||
|
limit 1;
|
||||||
|
"
|
||||||
|
| first
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def existing-active-job [note_id: string, source_hash: string] {
|
||||||
|
sql-json $"
|
||||||
|
select job_id
|
||||||
|
from jobs
|
||||||
|
where note_id = (sql-quote $note_id)
|
||||||
|
and source_hash = (sql-quote $source_hash)
|
||||||
|
and status != 'done'
|
||||||
|
and status != 'failed'
|
||||||
|
order by requested_at desc
|
||||||
|
limit 1;
|
||||||
|
"
|
||||||
|
| first
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def archive-current-source [note: record] {
|
||||||
|
if not ($note.source_path | path exists) {
|
||||||
|
error make {
|
||||||
|
msg: $"Current source path is missing: ($note.source_path)"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let source_hash = (sha256 $note.source_path)
|
||||||
|
let source_size = (((ls -l $note.source_path | first).size) | into int)
|
||||||
|
let source_mtime = (((ls -l $note.source_path | first).modified) | format date "%Y-%m-%dT%H:%M:%SZ")
|
||||||
|
let version = (archive-and-version $note.note_id $note.source_path $note.source_relpath $source_size $source_mtime $source_hash)
|
||||||
|
|
||||||
|
sql-run $"
|
||||||
|
update notes
|
||||||
|
set current_source_hash = (sql-quote $source_hash),
|
||||||
|
current_source_size = ($source_size),
|
||||||
|
current_source_mtime = (sql-quote $source_mtime),
|
||||||
|
current_archive_path = (sql-quote $version.archive_path),
|
||||||
|
latest_version_id = (sql-quote $version.version_id),
|
||||||
|
last_seen_at = (sql-quote (now-iso)),
|
||||||
|
status = 'active',
|
||||||
|
missing_since = null,
|
||||||
|
deleted_at = null
|
||||||
|
where note_id = (sql-quote $note.note_id);
|
||||||
|
"
|
||||||
|
| ignore
|
||||||
|
|
||||||
|
{
|
||||||
|
input_path: $version.archive_path
|
||||||
|
archive_path: $version.archive_path
|
||||||
|
source_hash: $source_hash
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def enqueue-reingest-job [note: record, source_hash: string, input_path: string, archive_path: string, force_overwrite_generated: bool] {
|
||||||
|
let job = (enqueue-job $note 'reingest' $input_path $archive_path $source_hash $note.title $force_overwrite_generated)
|
||||||
|
if $job == null {
|
||||||
|
let existing = (existing-active-job $note.note_id $source_hash)
|
||||||
|
print $"Already queued: ($existing.job_id? | default 'unknown')"
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
log-event $note.note_id 'reingest-enqueued' {
|
||||||
|
job_id: $job.job_id
|
||||||
|
source_hash: $source_hash
|
||||||
|
archive_path: $archive_path
|
||||||
|
force_overwrite_generated: $force_overwrite_generated
|
||||||
|
}
|
||||||
|
|
||||||
|
print $"Enqueued ($job.job_id) for ($note.note_id)"
|
||||||
|
|
||||||
|
try {
|
||||||
|
worker-run --drain
|
||||||
|
} catch {|error|
|
||||||
|
error make {
|
||||||
|
msg: (($error.msg? | default ($error | to nuon)) | into string)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def main [note_id: string, --latest-source, --latest-archive, --force-overwrite-generated] {
|
||||||
|
ensure-layout
|
||||||
|
|
||||||
|
let note_row = (sql-json $"
|
||||||
|
select *
|
||||||
|
from notes
|
||||||
|
where note_id = (sql-quote $note_id)
|
||||||
|
limit 1;
|
||||||
|
" | first)
|
||||||
|
let note = if $note_row == null {
|
||||||
|
null
|
||||||
|
} else {
|
||||||
|
$note_row | upsert source_path ([ (webdav-root) $note_row.source_relpath ] | path join)
|
||||||
|
}
|
||||||
|
|
||||||
|
if $note == null {
|
||||||
|
error make {
|
||||||
|
msg: $"Unknown note id: ($note_id)"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if $latest_source and $latest_archive {
|
||||||
|
error make {
|
||||||
|
msg: 'Choose only one of --latest-source or --latest-archive'
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let source_mode = if $latest_source {
|
||||||
|
'source'
|
||||||
|
} else if $latest_archive {
|
||||||
|
'archive'
|
||||||
|
} else if ($note.status == 'active' and ($note.source_path | path exists)) {
|
||||||
|
'source'
|
||||||
|
} else {
|
||||||
|
'archive'
|
||||||
|
}
|
||||||
|
|
||||||
|
if $source_mode == 'source' {
|
||||||
|
let archived = (archive-current-source $note)
|
||||||
|
enqueue-reingest-job $note $archived.source_hash $archived.input_path $archived.archive_path $force_overwrite_generated
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
let version = (latest-version $note.note_id)
|
||||||
|
if $version == null {
|
||||||
|
error make {
|
||||||
|
msg: $"No archived version found for ($note.note_id)"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
enqueue-reingest-job $note $version.source_hash $version.archive_path $version.archive_path $force_overwrite_generated
|
||||||
|
}
|
||||||
202
modules/hosts/_parts/tahani/notability/status.nu
Normal file
202
modules/hosts/_parts/tahani/notability/status.nu
Normal file
@@ -0,0 +1,202 @@
|
|||||||
|
#!/usr/bin/env nu
|
||||||
|
|
||||||
|
use ./lib.nu *
|
||||||
|
|
||||||
|
|
||||||
|
def format-summary [] {
|
||||||
|
let counts = (sql-json '
|
||||||
|
select status, count(*) as count
|
||||||
|
from notes
|
||||||
|
group by status
|
||||||
|
order by status;
|
||||||
|
')
|
||||||
|
let queue = (sql-json "
|
||||||
|
select status, count(*) as count
|
||||||
|
from jobs
|
||||||
|
where status in ('queued', 'running', 'failed')
|
||||||
|
group by status
|
||||||
|
order by status;
|
||||||
|
")
|
||||||
|
|
||||||
|
let lines = [
|
||||||
|
$"notes db: (db-path)"
|
||||||
|
$"webdav root: (webdav-root)"
|
||||||
|
$"notes root: (notes-root)"
|
||||||
|
''
|
||||||
|
'notes:'
|
||||||
|
]
|
||||||
|
|
||||||
|
let note_statuses = ('active,source_missing,source_deleted,conflict,failed' | split row ',')
|
||||||
|
let note_lines = (
|
||||||
|
$note_statuses
|
||||||
|
| each {|status|
|
||||||
|
let row = ($counts | where {|row| ($row | get 'status') == $status } | first)
|
||||||
|
let count = ($row.count? | default 0)
|
||||||
|
$" ($status): ($count)"
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
let job_statuses = ('queued,running,failed' | split row ',')
|
||||||
|
let job_lines = (
|
||||||
|
$job_statuses
|
||||||
|
| each {|status|
|
||||||
|
let row = ($queue | where {|row| ($row | get 'status') == $status } | first)
|
||||||
|
let count = ($row.count? | default 0)
|
||||||
|
$" ($status): ($count)"
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
($lines ++ $note_lines ++ ['' 'jobs:'] ++ $job_lines ++ ['']) | str join "\n"
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def format-note [note_id: string] {
|
||||||
|
let note = (sql-json $"
|
||||||
|
select *
|
||||||
|
from notes
|
||||||
|
where note_id = (sql-quote $note_id)
|
||||||
|
limit 1;
|
||||||
|
" | first)
|
||||||
|
|
||||||
|
if $note == null {
|
||||||
|
error make {
|
||||||
|
msg: $"Unknown note id: ($note_id)"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let jobs = (sql-json $"
|
||||||
|
select job_id, operation, status, requested_at, started_at, finished_at, source_hash, error_summary
|
||||||
|
from jobs
|
||||||
|
where note_id = (sql-quote $note_id)
|
||||||
|
order by requested_at desc
|
||||||
|
limit 5;
|
||||||
|
")
|
||||||
|
let events = (sql-json $"
|
||||||
|
select ts, kind, details
|
||||||
|
from events
|
||||||
|
where note_id = (sql-quote $note_id)
|
||||||
|
order by ts desc
|
||||||
|
limit 10;
|
||||||
|
")
|
||||||
|
let output_exists = ($note.output_path | path exists)
|
||||||
|
let frontmatter = (parse-output-frontmatter $note.output_path)
|
||||||
|
|
||||||
|
let lines = [
|
||||||
|
$"note_id: ($note.note_id)"
|
||||||
|
$"title: ($note.title)"
|
||||||
|
$"status: ($note.status)"
|
||||||
|
$"source_relpath: ($note.source_relpath)"
|
||||||
|
$"output_path: ($note.output_path)"
|
||||||
|
$"output_exists: ($output_exists)"
|
||||||
|
$"managed_by: ($frontmatter.managed_by? | default '')"
|
||||||
|
$"frontmatter_note_id: ($frontmatter.note_id? | default '')"
|
||||||
|
$"current_source_hash: ($note.current_source_hash? | default '')"
|
||||||
|
$"last_generated_output_hash: ($note.last_generated_output_hash? | default '')"
|
||||||
|
$"current_archive_path: ($note.current_archive_path? | default '')"
|
||||||
|
$"last_processed_at: ($note.last_processed_at? | default '')"
|
||||||
|
$"missing_since: ($note.missing_since? | default '')"
|
||||||
|
$"deleted_at: ($note.deleted_at? | default '')"
|
||||||
|
$"conflict_path: ($note.conflict_path? | default '')"
|
||||||
|
$"last_error: ($note.last_error? | default '')"
|
||||||
|
''
|
||||||
|
'recent jobs:'
|
||||||
|
]
|
||||||
|
|
||||||
|
let job_lines = if ($jobs | is-empty) {
|
||||||
|
[' (none)']
|
||||||
|
} else {
|
||||||
|
$jobs | each {|job|
|
||||||
|
$" ($job.job_id) [($job.status)] ($job.operation) requested=($job.requested_at) error=($job.error_summary? | default '')"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let event_lines = if ($events | is-empty) {
|
||||||
|
[' (none)']
|
||||||
|
} else {
|
||||||
|
$events | each {|event|
|
||||||
|
$" ($event.ts) ($event.kind) ($event.details? | default '')"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
($lines ++ $job_lines ++ ['' 'recent events:'] ++ $event_lines ++ ['']) | str join "\n"
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def format-filtered [status: string, label: string] {
|
||||||
|
let notes = (sql-json $"
|
||||||
|
select note_id, title, source_relpath, output_path, status, last_error, conflict_path
|
||||||
|
from notes
|
||||||
|
where status = (sql-quote $status)
|
||||||
|
order by last_seen_at desc;
|
||||||
|
")
|
||||||
|
|
||||||
|
let header = [$label]
|
||||||
|
let body = if ($notes | is-empty) {
|
||||||
|
[' (none)']
|
||||||
|
} else {
|
||||||
|
$notes | each {|note|
|
||||||
|
let extra = if $status == 'conflict' {
|
||||||
|
$" conflict_path=($note.conflict_path? | default '')"
|
||||||
|
} else if $status == 'failed' {
|
||||||
|
$" last_error=($note.last_error? | default '')"
|
||||||
|
} else {
|
||||||
|
''
|
||||||
|
}
|
||||||
|
$" ($note.note_id) ($note.title) [($note.status)] source=($note.source_relpath) output=($note.output_path)($extra)"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
($header ++ $body ++ ['']) | str join "\n"
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def format-queue [] {
|
||||||
|
let jobs = (sql-json "
|
||||||
|
select job_id, note_id, operation, status, requested_at, started_at, error_summary
|
||||||
|
from jobs
|
||||||
|
where status in ('queued', 'running', 'failed')
|
||||||
|
order by requested_at asc;
|
||||||
|
")
|
||||||
|
|
||||||
|
let lines = if ($jobs | is-empty) {
|
||||||
|
['queue' ' (empty)' '']
|
||||||
|
} else {
|
||||||
|
['queue'] ++ ($jobs | each {|job|
|
||||||
|
$" ($job.job_id) note=($job.note_id) [($job.status)] ($job.operation) requested=($job.requested_at) error=($job.error_summary? | default '')"
|
||||||
|
}) ++ ['']
|
||||||
|
}
|
||||||
|
|
||||||
|
$lines | str join "\n"
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def main [note_id?: string, --failed, --queue, --deleted, --conflicts] {
|
||||||
|
ensure-layout
|
||||||
|
|
||||||
|
if $queue {
|
||||||
|
print (format-queue)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if $failed {
|
||||||
|
print (format-filtered 'failed' 'failed notes')
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if $deleted {
|
||||||
|
print (format-filtered 'source_deleted' 'deleted notes')
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if $conflicts {
|
||||||
|
print (format-filtered 'conflict' 'conflict notes')
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if $note_id != null {
|
||||||
|
print (format-note $note_id)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
print (format-summary)
|
||||||
|
}
|
||||||
58
modules/hosts/_parts/tahani/notability/watch.nu
Normal file
58
modules/hosts/_parts/tahani/notability/watch.nu
Normal file
@@ -0,0 +1,58 @@
|
|||||||
|
#!/usr/bin/env nu
|
||||||
|
|
||||||
|
use ./lib.nu *
|
||||||
|
use ./reconcile.nu [reconcile-run]
|
||||||
|
use ./worker.nu [worker-run]
|
||||||
|
|
||||||
|
|
||||||
|
def error-message [error: any] {
|
||||||
|
let msg = (($error.msg? | default '') | into string)
|
||||||
|
if $msg == '' {
|
||||||
|
$error | to nuon
|
||||||
|
} else {
|
||||||
|
$msg
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def run-worker [] {
|
||||||
|
try {
|
||||||
|
worker-run --drain
|
||||||
|
} catch {|error|
|
||||||
|
print $"worker failed: (error-message $error)"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def run-sync [] {
|
||||||
|
run-worker
|
||||||
|
|
||||||
|
try {
|
||||||
|
reconcile-run
|
||||||
|
} catch {|error|
|
||||||
|
print $"reconcile failed: (error-message $error)"
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
run-worker
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def main [] {
|
||||||
|
ensure-layout
|
||||||
|
let root = (webdav-root)
|
||||||
|
print $"Watching ($root) for Notability WebDAV updates"
|
||||||
|
|
||||||
|
run-sync
|
||||||
|
|
||||||
|
^inotifywait -m -r --format '%w%f' -e create -e close_write -e moved_to -e moved_from -e delete -e attrib $root
|
||||||
|
| lines
|
||||||
|
| each {|changed_path|
|
||||||
|
if not (is-supported-source-path $changed_path) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
print $"Filesystem event for ($changed_path)"
|
||||||
|
run-sync
|
||||||
|
}
|
||||||
|
}
|
||||||
36
modules/hosts/_parts/tahani/notability/webdav.nu
Normal file
36
modules/hosts/_parts/tahani/notability/webdav.nu
Normal file
@@ -0,0 +1,36 @@
|
|||||||
|
#!/usr/bin/env nu
|
||||||
|
|
||||||
|
use ./lib.nu *
|
||||||
|
|
||||||
|
|
||||||
|
def main [] {
|
||||||
|
ensure-layout
|
||||||
|
|
||||||
|
let root = (webdav-root)
|
||||||
|
let addr = if ('NOTABILITY_WEBDAV_ADDR' in ($env | columns)) {
|
||||||
|
$env.NOTABILITY_WEBDAV_ADDR
|
||||||
|
} else {
|
||||||
|
'127.0.0.1:9980'
|
||||||
|
}
|
||||||
|
let user = if ('NOTABILITY_WEBDAV_USER' in ($env | columns)) {
|
||||||
|
$env.NOTABILITY_WEBDAV_USER
|
||||||
|
} else {
|
||||||
|
'notability'
|
||||||
|
}
|
||||||
|
let baseurl = if ('NOTABILITY_WEBDAV_BASEURL' in ($env | columns)) {
|
||||||
|
$env.NOTABILITY_WEBDAV_BASEURL
|
||||||
|
} else {
|
||||||
|
'/'
|
||||||
|
}
|
||||||
|
let password_file = if ('NOTABILITY_WEBDAV_PASSWORD_FILE' in ($env | columns)) {
|
||||||
|
$env.NOTABILITY_WEBDAV_PASSWORD_FILE
|
||||||
|
} else {
|
||||||
|
error make {
|
||||||
|
msg: 'NOTABILITY_WEBDAV_PASSWORD_FILE is required'
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let password = (open --raw $password_file | str trim)
|
||||||
|
|
||||||
|
print $"Starting WebDAV on ($addr), serving ($root), base URL ($baseurl)"
|
||||||
|
run-external rclone 'serve' 'webdav' $root '--addr' $addr '--baseurl' $baseurl '--user' $user '--pass' $password
|
||||||
|
}
|
||||||
506
modules/hosts/_parts/tahani/notability/worker.nu
Normal file
506
modules/hosts/_parts/tahani/notability/worker.nu
Normal file
@@ -0,0 +1,506 @@
|
|||||||
|
#!/usr/bin/env nu
|
||||||
|
|
||||||
|
use ./lib.nu *
|
||||||
|
|
||||||
|
const qmd_debounce = 1min
|
||||||
|
const idle_sleep = 10sec
|
||||||
|
const vision_model = 'openai-codex/gpt-5.4'
|
||||||
|
const transcribe_timeout = '90s'
|
||||||
|
const normalize_timeout = '60s'
|
||||||
|
|
||||||
|
|
||||||
|
def next-queued-job [] {
|
||||||
|
sql-json "
|
||||||
|
select job_id, note_id, operation, job_manifest_path, result_path, source_hash
|
||||||
|
from jobs
|
||||||
|
where status = 'queued'
|
||||||
|
order by requested_at asc
|
||||||
|
limit 1;
|
||||||
|
"
|
||||||
|
| first
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def maybe-update-qmd [] {
|
||||||
|
let dirty = (qmd-dirty-file)
|
||||||
|
if not ($dirty | path exists) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
let modified = ((ls -l $dirty | first).modified)
|
||||||
|
if ((date now) - $modified) < $qmd_debounce {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
print 'Running qmd update'
|
||||||
|
let result = (do {
|
||||||
|
cd (notes-root)
|
||||||
|
run-external qmd 'update' | complete
|
||||||
|
})
|
||||||
|
if $result.exit_code != 0 {
|
||||||
|
print $"qmd update failed: ($result.stderr | str trim)"
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
rm -f $dirty
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def write-result [result_path: path, payload: record] {
|
||||||
|
mkdir ($result_path | path dirname)
|
||||||
|
($payload | to json --indent 2) | save -f $result_path
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def error-message [error: any] {
|
||||||
|
let msg = (($error.msg? | default '') | into string)
|
||||||
|
if ($msg == '' or $msg == 'External command failed') {
|
||||||
|
$error | to nuon
|
||||||
|
} else {
|
||||||
|
$msg
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def unquote [value?: any] {
|
||||||
|
if $value == null {
|
||||||
|
''
|
||||||
|
} else {
|
||||||
|
($value | into string | str replace -r '^"(.*)"$' '$1' | str replace -r "^'(.*)'$" '$1')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def source-format [file: path] {
|
||||||
|
(([$file] | path parse | first).extension? | default 'bin' | str downcase)
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def conflict-path-for [output_path: path] {
|
||||||
|
let parsed = ([$output_path] | path parse | first)
|
||||||
|
let stamp = ((date now) | format date '%Y-%m-%dT%H-%M-%SZ')
|
||||||
|
[$parsed.parent $"($parsed.stem).conflict-($stamp).($parsed.extension)"] | path join
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def find-managed-outputs [note_id: string] {
|
||||||
|
let root = (notes-root)
|
||||||
|
if not ($root | path exists) {
|
||||||
|
[]
|
||||||
|
} else {
|
||||||
|
(glob $"($root)/**/*.md")
|
||||||
|
| where not ($it | str contains '/.')
|
||||||
|
| where {|file|
|
||||||
|
let parsed = (parse-output-frontmatter $file)
|
||||||
|
(unquote ($parsed.managed_by? | default '')) == 'notability-ingest' and (unquote ($parsed.note_id? | default '')) == $note_id
|
||||||
|
}
|
||||||
|
| sort
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def resolve-managed-output-path [note_id: string, configured_output_path: path] {
|
||||||
|
if ($configured_output_path | path exists) {
|
||||||
|
let parsed = (parse-output-frontmatter $configured_output_path)
|
||||||
|
let managed_by = (unquote ($parsed.managed_by? | default ''))
|
||||||
|
let frontmatter_note_id = (unquote ($parsed.note_id? | default ''))
|
||||||
|
if ($managed_by == 'notability-ingest' and $frontmatter_note_id == $note_id) {
|
||||||
|
return $configured_output_path
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let discovered = (find-managed-outputs $note_id)
|
||||||
|
if ($discovered | is-empty) {
|
||||||
|
$configured_output_path
|
||||||
|
} else if (($discovered | length) == 1) {
|
||||||
|
$discovered | first
|
||||||
|
} else {
|
||||||
|
error make {
|
||||||
|
msg: $"Multiple managed note files found for ($note_id): (($discovered | str join ', '))"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def determine-write-target [manifest: record] {
|
||||||
|
let output_path = (resolve-managed-output-path $manifest.note_id $manifest.output_path)
|
||||||
|
if not ($output_path | path exists) {
|
||||||
|
return {
|
||||||
|
output_path: $output_path
|
||||||
|
write_path: $output_path
|
||||||
|
write_mode: 'create'
|
||||||
|
updated_main_output: true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let parsed = (parse-output-frontmatter $output_path)
|
||||||
|
let managed_by = (unquote ($parsed.managed_by? | default ''))
|
||||||
|
let frontmatter_note_id = (unquote ($parsed.note_id? | default ''))
|
||||||
|
if ($managed_by == 'notability-ingest' and $frontmatter_note_id == $manifest.note_id) {
|
||||||
|
return {
|
||||||
|
output_path: $output_path
|
||||||
|
write_path: $output_path
|
||||||
|
write_mode: 'overwrite'
|
||||||
|
updated_main_output: true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
{
|
||||||
|
output_path: $output_path
|
||||||
|
write_path: (conflict-path-for $output_path)
|
||||||
|
write_mode: 'conflict'
|
||||||
|
updated_main_output: false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def build-markdown [manifest: record, normalized: string] {
|
||||||
|
let body = ($normalized | str trim)
|
||||||
|
let output_body = if $body == '' {
|
||||||
|
$"# ($manifest.title)"
|
||||||
|
} else {
|
||||||
|
$body
|
||||||
|
}
|
||||||
|
let created = ($manifest.requested_at | str substring 0..9)
|
||||||
|
let updated = ((date now) | format date '%Y-%m-%d')
|
||||||
|
|
||||||
|
[
|
||||||
|
'---'
|
||||||
|
$"title: ($manifest.title | to json)"
|
||||||
|
$"created: ($created | to json)"
|
||||||
|
$"updated: ($updated | to json)"
|
||||||
|
'source: "notability"'
|
||||||
|
$"source_transport: (($manifest.source_transport? | default 'webdav') | to json)"
|
||||||
|
$"source_relpath: ($manifest.source_relpath | to json)"
|
||||||
|
$"note_id: ($manifest.note_id | to json)"
|
||||||
|
'managed_by: "notability-ingest"'
|
||||||
|
$"source_file: ($manifest.archive_path | to json)"
|
||||||
|
$"source_file_hash: ($'sha256:($manifest.source_hash)' | to json)"
|
||||||
|
$"source_format: ((source-format $manifest.archive_path) | to json)"
|
||||||
|
'status: "active"'
|
||||||
|
'tags:'
|
||||||
|
' - handwritten'
|
||||||
|
' - notability'
|
||||||
|
'---'
|
||||||
|
''
|
||||||
|
$output_body
|
||||||
|
''
|
||||||
|
] | str join "\n"
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def render-pages [input_path: path, job_id: string] {
|
||||||
|
let extension = (([$input_path] | path parse | first).extension? | default '' | str downcase)
|
||||||
|
if $extension == 'png' {
|
||||||
|
[ $input_path ]
|
||||||
|
} else if $extension == 'pdf' {
|
||||||
|
let render_dir = [(render-root) $job_id] | path join
|
||||||
|
mkdir $render_dir
|
||||||
|
let prefix = [$render_dir 'page'] | path join
|
||||||
|
^pdftoppm -png -r 200 $input_path $prefix
|
||||||
|
let pages = ((glob $"($render_dir)/*.png") | sort)
|
||||||
|
if ($pages | is-empty) {
|
||||||
|
error make {
|
||||||
|
msg: $"No PNG pages rendered from ($input_path)"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
$pages
|
||||||
|
} else {
|
||||||
|
error make {
|
||||||
|
msg: $"Unsupported Notability input format: ($input_path)"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def call-pi [timeout_window: string, prompt: string, inputs: list<path>, thinking: string] {
|
||||||
|
let prompt_file = (^mktemp --suffix '.md' | str trim)
|
||||||
|
$prompt | save -f $prompt_file
|
||||||
|
let input_refs = ($inputs | each {|input| $'@($input)' })
|
||||||
|
let prompt_ref = $'@($prompt_file)'
|
||||||
|
let result = (try {
|
||||||
|
^timeout $timeout_window pi --model $vision_model --thinking $thinking --no-tools --no-session -p ...$input_refs $prompt_ref | complete
|
||||||
|
} catch {|error|
|
||||||
|
rm -f $prompt_file
|
||||||
|
error make {
|
||||||
|
msg: (error-message $error)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
rm -f $prompt_file
|
||||||
|
|
||||||
|
let output = ($result.stdout | str trim)
|
||||||
|
if $output != '' {
|
||||||
|
$output
|
||||||
|
} else {
|
||||||
|
let stderr = ($result.stderr | str trim)
|
||||||
|
if $stderr == '' {
|
||||||
|
error make {
|
||||||
|
msg: $"pi returned no output (exit ($result.exit_code))"
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
error make {
|
||||||
|
msg: $"pi returned no output (exit ($result.exit_code)): ($stderr)"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def ingest-job [manifest: record] {
|
||||||
|
mkdir $manifest.session_dir
|
||||||
|
|
||||||
|
let page_paths = (render-pages $manifest.input_path $manifest.job_id)
|
||||||
|
let transcribe_prompt = ([
|
||||||
|
'Transcribe this note into clean Markdown.'
|
||||||
|
''
|
||||||
|
'Read it like a human and reconstruct the intended reading order and structure.'
|
||||||
|
''
|
||||||
|
'Do not preserve handwritten layout literally.'
|
||||||
|
''
|
||||||
|
'Handwritten line breaks, word stacking, font size changes, and spacing are not semantic structure by default.'
|
||||||
|
''
|
||||||
|
'If adjacent handwritten lines clearly belong to one sentence or short phrase, merge them into normal prose with spaces instead of separate Markdown lines.'
|
||||||
|
''
|
||||||
|
'Only keep separate lines or blank lines when there is clear evidence of separate paragraphs, headings, list items, checkboxes, or other distinct blocks.'
|
||||||
|
''
|
||||||
|
'Keep headings, lists, and paragraphs when they are genuinely present.'
|
||||||
|
''
|
||||||
|
'Do not summarize. Do not add commentary. Return Markdown only.'
|
||||||
|
] | str join "\n")
|
||||||
|
print $"Transcribing ($manifest.job_id) with page count ($page_paths | length)"
|
||||||
|
let transcript = (call-pi $transcribe_timeout $transcribe_prompt $page_paths 'low')
|
||||||
|
mkdir ($manifest.transcript_path | path dirname)
|
||||||
|
$"($transcript)\n" | save -f $manifest.transcript_path
|
||||||
|
|
||||||
|
let normalize_prompt = ([
|
||||||
|
'Rewrite the attached transcription into clean Markdown.'
|
||||||
|
''
|
||||||
|
'Preserve the same content and intended structure.'
|
||||||
|
''
|
||||||
|
'Collapse layout-only line breaks from handwriting.'
|
||||||
|
''
|
||||||
|
'If short adjacent lines are really one sentence or phrase, join them with spaces instead of keeping one line per handwritten row.'
|
||||||
|
''
|
||||||
|
'Use separate lines only for real headings, list items, checkboxes, or distinct paragraphs.'
|
||||||
|
''
|
||||||
|
'Do not summarize. Return Markdown only.'
|
||||||
|
] | str join "\n")
|
||||||
|
print $"Normalizing ($manifest.job_id)"
|
||||||
|
let normalized = (call-pi $normalize_timeout $normalize_prompt [ $manifest.transcript_path ] 'off')
|
||||||
|
|
||||||
|
let markdown = (build-markdown $manifest $normalized)
|
||||||
|
let target = (determine-write-target $manifest)
|
||||||
|
mkdir ($target.write_path | path dirname)
|
||||||
|
$markdown | save -f $target.write_path
|
||||||
|
|
||||||
|
{
|
||||||
|
success: true
|
||||||
|
job_id: $manifest.job_id
|
||||||
|
note_id: $manifest.note_id
|
||||||
|
archive_path: $manifest.archive_path
|
||||||
|
source_hash: $manifest.source_hash
|
||||||
|
session_dir: $manifest.session_dir
|
||||||
|
output_path: $target.output_path
|
||||||
|
output_hash: (if $target.updated_main_output { sha256 $target.write_path } else { null })
|
||||||
|
conflict_path: (if $target.write_mode == 'conflict' { $target.write_path } else { null })
|
||||||
|
write_mode: $target.write_mode
|
||||||
|
updated_main_output: $target.updated_main_output
|
||||||
|
transcript_path: $manifest.transcript_path
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def mark-failure [job: record, running_path: string, error_summary: string, result?: any] {
|
||||||
|
let finished_at = (now-iso)
|
||||||
|
sql-run $"
|
||||||
|
update jobs
|
||||||
|
set status = 'failed',
|
||||||
|
finished_at = (sql-quote $finished_at),
|
||||||
|
error_summary = (sql-quote $error_summary),
|
||||||
|
job_manifest_path = (sql-quote (manifest-path-for $job.job_id 'failed'))
|
||||||
|
where job_id = (sql-quote $job.job_id);
|
||||||
|
|
||||||
|
update notes
|
||||||
|
set status = 'failed',
|
||||||
|
last_error = (sql-quote $error_summary)
|
||||||
|
where note_id = (sql-quote $job.note_id);
|
||||||
|
"
|
||||||
|
| ignore
|
||||||
|
|
||||||
|
if $result != null and ($result.archive_path? | default null) != null {
|
||||||
|
sql-run $"
|
||||||
|
update versions
|
||||||
|
set ingest_result = 'failed',
|
||||||
|
session_path = (sql-quote ($result.session_dir? | default ''))
|
||||||
|
where archive_path = (sql-quote $result.archive_path);
|
||||||
|
"
|
||||||
|
| ignore
|
||||||
|
}
|
||||||
|
|
||||||
|
let failed_path = (manifest-path-for $job.job_id 'failed')
|
||||||
|
if ($running_path | path exists) {
|
||||||
|
mv -f $running_path $failed_path
|
||||||
|
}
|
||||||
|
|
||||||
|
log-event $job.note_id 'job-failed' {
|
||||||
|
job_id: $job.job_id
|
||||||
|
error: $error_summary
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def mark-success [job: record, running_path: string, result: record] {
|
||||||
|
let finished_at = (now-iso)
|
||||||
|
let note_status = if ($result.write_mode? | default 'write') == 'conflict' {
|
||||||
|
'conflict'
|
||||||
|
} else {
|
||||||
|
'active'
|
||||||
|
}
|
||||||
|
let output_path_q = (sql-quote ($result.output_path? | default null))
|
||||||
|
let output_hash_update = if ($result.updated_main_output? | default false) {
|
||||||
|
sql-quote ($result.output_hash? | default null)
|
||||||
|
} else {
|
||||||
|
'last_generated_output_hash'
|
||||||
|
}
|
||||||
|
let source_hash_update = if ($result.updated_main_output? | default false) {
|
||||||
|
sql-quote ($result.source_hash? | default null)
|
||||||
|
} else {
|
||||||
|
'last_generated_source_hash'
|
||||||
|
}
|
||||||
|
|
||||||
|
sql-run $"
|
||||||
|
update jobs
|
||||||
|
set status = 'done',
|
||||||
|
finished_at = (sql-quote $finished_at),
|
||||||
|
error_summary = null,
|
||||||
|
job_manifest_path = (sql-quote (manifest-path-for $job.job_id 'done'))
|
||||||
|
where job_id = (sql-quote $job.job_id);
|
||||||
|
|
||||||
|
update notes
|
||||||
|
set status = (sql-quote $note_status),
|
||||||
|
output_path = ($output_path_q),
|
||||||
|
last_processed_at = (sql-quote $finished_at),
|
||||||
|
last_generated_output_hash = ($output_hash_update),
|
||||||
|
last_generated_source_hash = ($source_hash_update),
|
||||||
|
conflict_path = (sql-quote ($result.conflict_path? | default null)),
|
||||||
|
last_error = null
|
||||||
|
where note_id = (sql-quote $job.note_id);
|
||||||
|
|
||||||
|
update versions
|
||||||
|
set ingest_result = 'success',
|
||||||
|
session_path = (sql-quote ($result.session_dir? | default ''))
|
||||||
|
where archive_path = (sql-quote $result.archive_path);
|
||||||
|
"
|
||||||
|
| ignore
|
||||||
|
|
||||||
|
let done_path = (manifest-path-for $job.job_id 'done')
|
||||||
|
if ($running_path | path exists) {
|
||||||
|
mv -f $running_path $done_path
|
||||||
|
}
|
||||||
|
|
||||||
|
^touch (qmd-dirty-file)
|
||||||
|
|
||||||
|
log-event $job.note_id 'job-finished' {
|
||||||
|
job_id: $job.job_id
|
||||||
|
write_mode: ($result.write_mode? | default 'write')
|
||||||
|
output_path: ($result.output_path? | default '')
|
||||||
|
conflict_path: ($result.conflict_path? | default '')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def recover-running-jobs [] {
|
||||||
|
let jobs = (sql-json "
|
||||||
|
select job_id, note_id, job_manifest_path, result_path
|
||||||
|
from jobs
|
||||||
|
where status = 'running'
|
||||||
|
order by started_at asc;
|
||||||
|
")
|
||||||
|
|
||||||
|
for job in $jobs {
|
||||||
|
let running_path = (manifest-path-for $job.job_id 'running')
|
||||||
|
let result = if ($job.result_path | path exists) {
|
||||||
|
open $job.result_path
|
||||||
|
} else {
|
||||||
|
null
|
||||||
|
}
|
||||||
|
mark-failure $job $running_path 'worker interrupted before completion' $result
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def process-job [job: record] {
|
||||||
|
let running_path = (manifest-path-for $job.job_id 'running')
|
||||||
|
mv -f $job.job_manifest_path $running_path
|
||||||
|
sql-run $"
|
||||||
|
update jobs
|
||||||
|
set status = 'running',
|
||||||
|
started_at = (sql-quote (now-iso)),
|
||||||
|
job_manifest_path = (sql-quote $running_path)
|
||||||
|
where job_id = (sql-quote $job.job_id);
|
||||||
|
"
|
||||||
|
| ignore
|
||||||
|
|
||||||
|
print $"Processing ($job.job_id) for ($job.note_id)"
|
||||||
|
|
||||||
|
let manifest = (open $running_path)
|
||||||
|
try {
|
||||||
|
let result = (ingest-job $manifest)
|
||||||
|
write-result $job.result_path $result
|
||||||
|
mark-success $job $running_path $result
|
||||||
|
} catch {|error|
|
||||||
|
let message = (error-message $error)
|
||||||
|
let result = {
|
||||||
|
success: false
|
||||||
|
job_id: $manifest.job_id
|
||||||
|
note_id: $manifest.note_id
|
||||||
|
archive_path: $manifest.archive_path
|
||||||
|
source_hash: $manifest.source_hash
|
||||||
|
session_dir: $manifest.session_dir
|
||||||
|
error: $message
|
||||||
|
}
|
||||||
|
write-result $job.result_path $result
|
||||||
|
mark-failure $job $running_path $message $result
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def drain-queued-jobs [] {
|
||||||
|
loop {
|
||||||
|
let job = (next-queued-job)
|
||||||
|
if $job == null {
|
||||||
|
maybe-update-qmd
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
process-job $job
|
||||||
|
maybe-update-qmd
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
export def worker-run [--drain] {
|
||||||
|
ensure-layout
|
||||||
|
recover-running-jobs
|
||||||
|
if $drain {
|
||||||
|
drain-queued-jobs
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
while true {
|
||||||
|
let job = (next-queued-job)
|
||||||
|
if $job == null {
|
||||||
|
maybe-update-qmd
|
||||||
|
sleep $idle_sleep
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
process-job $job
|
||||||
|
maybe-update-qmd
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def main [--drain] {
|
||||||
|
worker-run --drain=$drain
|
||||||
|
}
|
||||||
@@ -31,6 +31,7 @@
|
|||||||
./_parts/tahani/adguardhome.nix
|
./_parts/tahani/adguardhome.nix
|
||||||
./_parts/tahani/cache.nix
|
./_parts/tahani/cache.nix
|
||||||
./_parts/tahani/networking.nix
|
./_parts/tahani/networking.nix
|
||||||
|
./_parts/tahani/notability.nix
|
||||||
./_parts/tahani/paperless.nix
|
./_parts/tahani/paperless.nix
|
||||||
];
|
];
|
||||||
|
|
||||||
|
|||||||
@@ -26,6 +26,8 @@
|
|||||||
(import ./_overlays/pi-harness.nix {inherit inputs;})
|
(import ./_overlays/pi-harness.nix {inherit inputs;})
|
||||||
# pi-mcp-adapter
|
# pi-mcp-adapter
|
||||||
(import ./_overlays/pi-mcp-adapter.nix {inherit inputs;})
|
(import ./_overlays/pi-mcp-adapter.nix {inherit inputs;})
|
||||||
|
# qmd
|
||||||
|
(import ./_overlays/qmd.nix {inherit inputs;})
|
||||||
# jj-starship (passes through upstream overlay)
|
# jj-starship (passes through upstream overlay)
|
||||||
(import ./_overlays/jj-starship.nix {inherit inputs;})
|
(import ./_overlays/jj-starship.nix {inherit inputs;})
|
||||||
# zjstatus
|
# zjstatus
|
||||||
|
|||||||
@@ -66,18 +66,20 @@
|
|||||||
'';
|
'';
|
||||||
|
|
||||||
xdg.configFile = {
|
xdg.configFile = {
|
||||||
"glow/glow.yml".text = ''
|
"glow/glow.yml".text =
|
||||||
# style name or JSON path (default "auto")
|
lib.concatStringsSep "\n" [
|
||||||
style: "${config.xdg.configHome}/glow/rose-pine-dawn.json"
|
"# style name or JSON path (default \"auto\")"
|
||||||
# mouse support (TUI-mode only)
|
"style: \"${config.xdg.configHome}/glow/rose-pine-dawn.json\""
|
||||||
mouse: false
|
"# mouse support (TUI-mode only)"
|
||||||
# use pager to display markdown
|
"mouse: false"
|
||||||
pager: false
|
"# use pager to display markdown"
|
||||||
# word-wrap at width
|
"pager: false"
|
||||||
width: 80
|
"# word-wrap at width"
|
||||||
# show all files, including hidden and ignored.
|
"width: 80"
|
||||||
all: false
|
"# show all files, including hidden and ignored."
|
||||||
'';
|
"all: false"
|
||||||
|
""
|
||||||
|
];
|
||||||
"glow/rose-pine-dawn.json".source = ./_terminal/rose-pine-dawn-glow.json;
|
"glow/rose-pine-dawn.json".source = ./_terminal/rose-pine-dawn-glow.json;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
30
secrets/tahani-notability-webdav-password
Normal file
30
secrets/tahani-notability-webdav-password
Normal file
@@ -0,0 +1,30 @@
|
|||||||
|
{
|
||||||
|
"data": "ENC[AES256_GCM,data:qZCh11bq1W7FwXMrDX5KMOQFsgsKgbhimZ4TDNvv1BDU,iv:PJJJB5uyhuTUSA4doQ6h6qMbmPgerPv+FfsJ0f20kYY=,tag:lXpit9T7K2rGUu1zsJH6dg==,type:str]",
|
||||||
|
"sops": {
|
||||||
|
"age": [
|
||||||
|
{
|
||||||
|
"recipient": "age1xate984yhl9qk9d4q99pyxmzz48sq56nfhu8weyzkgum4ed5tc5shjmrs7",
|
||||||
|
"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBvNXBjN3RZcGd2R2MyQWhR\nenBpa3VHMFhkWDEveUtmZWtPSk01QkhUVFJFCnRSc3ZGdFFjVDJnbHpwKzZ1TUdI\nZUdWQzM2bmZ1RUl4UVpCbDJoL0RkQncKLS0tIHRxUzFiaS8wekdCQ0Z0dTMxSnZ0\nS0UycFNMSUJHcVlkR2JZNlZsbldoaUkKe4EaYIquhABMEywizJXzEVEM1JbEwFqU\nAmQ6R+p4mNgaR5HCrnINQId3qqVfsP2UDqPDepERZIA0V2E5h9ckfQ==\n-----END AGE ENCRYPTED FILE-----\n"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"recipient": "age1njjegjjdqzfnrr54f536yl4lduqgna3wuv7ef6vtl9jw5cju0grsgy62tm",
|
||||||
|
"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBBZ1hYenpVTm1lTFdjTEJj\nTUN5MzNtbzdWNzQ2VE9tRlJJRVRYTUtLOXpnCnlLWTZPNGE5NDlwRHhWSnlTNUhv\nc3VZVklEZDB5dXlFc01wcEQxckl0NjgKLS0tIEE5T2JmNlJaYkZpWkhYdDhPSTlW\nei96YmhUWUZ2enVnRjhKOVlNZmNHa3cKxaHBtCwLDLNcscptlDk6ta/i491lLPt6\nOh/RtbkxtJ02cahIsKgajspOElx8u2Nb3/lmK51JbUIexH9TDQ+3tg==\n-----END AGE ENCRYPTED FILE-----\n"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"recipient": "age187jl7e4k9n4guygkmpuqzeh0wenefwrfkpvuyhvwjrjwxqpzassqq3x67j",
|
||||||
|
"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBJbFFpQzB2OU9jYUZlL2Nl\nOEZ0WGcyR1BpSmZGU0Vxa0N6WGpCbXBXZGxJCnlLK0JJWElndC9KRGN5d1NNd0tj\nUkExQ0tTSGRKQjJHUGtaWUtKS285MU0KLS0tIGI5cWtVcW43b2Q5VXRidllzamtB\nV1IxYnN1KzdaaXdvWG96a2VkZ0ZvWGsKxdbXwbgFIc3/3VjwUJ1A+cX0oaT+oojz\nrI9Dmk782U/dQrcMv1lRBIWWtAdAqS6GiQ1aUKk5aHpuHOZeHHFjMw==\n-----END AGE ENCRYPTED FILE-----\n"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"recipient": "age1ez6j3r5wdp0tjy7n5qzv5vfakdc2nh2zeu388zu7a80l0thv052syxq5e2",
|
||||||
|
"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA0aTgwQ3ZEVG41eW9MQ1RX\nSElRdkdvL21kZ2ZLeGNPbGJiNll5WjdsM2gwCmJQVmJjWEJBaVhEKzJqYWlib2JX\ndWRzSE9QTVQ1c004dldzR2NtR3pvQlUKLS0tIEsvZDNnNWJJaWZyOCtYUEs1eklh\nNXl2dUM0amVtSmdjTy83ZzBSeGp3Q0UKQ/cUYPACFNcxulzW964ftsHjoCBRGB66\nc1e/ObQNM+b+be5UzJi3/gago9CHRzZ3Rp6zE9i5oQBzgLGWlJuPNQ==\n-----END AGE ENCRYPTED FILE-----\n"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"recipient": "age1tlymdmaukhwupzrhszspp26lgd8s64rw4vu9lwc7gsgrjm78095s9fe9l3",
|
||||||
|
"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBLNUk5aHBqdEJoYWdaeVlx\nOUkrSXMvRFRmQ29QRE5hWTlHdlcwOUxFRXdRCnE0L1BQdHZDRWRCQUZ2dHQ2Witi\nQ1g5OFFWM2tPT0xEZUZvdXJNdm9aWTgKLS0tIENvM1h1V042L3JHV1pWeDAxdG84\nUTBTZjdHa1lCNGJSRG1iZmtpc1laZTQK/twptPseDi9DM/7NX2F0JO1BEkqklbh1\nxQ1Qwpy4K/P2pFTOBKqDb62DaIALxiGA1Q55dw+fPRSsnL8VcxG8JA==\n-----END AGE ENCRYPTED FILE-----\n"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"lastmodified": "2026-03-25T11:23:08Z",
|
||||||
|
"mac": "ENC[AES256_GCM,data:UM0QWfQueExEHRjqNAEIgwpVBjgpd0a6DXxDeRci08qMzTypTlWIofUGMyM1k+J+mUKr3vWMe3q48OwVtUaXnbWimH+8uFEwb5x0e+ayTg+w/C23d+JJmQIX8g5JXtknUAZFNrh3wdZOadYYRr/vDzCKud4lMrmFBKFXsH1DPEI=,iv:kTx8omo8Gt4mTLAs6MoLxj4GizWpxlSXMCTWNlRR5SY=,tag:PB7nMCVxCLRQdhC/eelK/w==,type:str]",
|
||||||
|
"version": "3.12.2"
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user