Compare commits

..

9 Commits

Author SHA1 Message Date
2d3e15231a flk 2026-03-26 07:56:26 +00:00
a6cc5dcc4a up 2026-03-26 07:56:26 +00:00
958c332bf1 up 2026-03-26 07:56:26 +00:00
49fa4d623e refactor notability ingest stack 2026-03-26 07:56:26 +00:00
4eefa6b337 feat(notes): add Notability WebDAV ingest pipeline 2026-03-26 07:56:26 +00:00
bef2afed66 flk 2026-03-26 07:56:26 +00:00
5ad97d97a7 up 2026-03-26 07:56:26 +00:00
51b0bd4b1d fix 2026-03-26 07:56:26 +00:00
19c770c163 fix 2026-03-26 07:56:26 +00:00
28 changed files with 8201 additions and 80 deletions

145
flake.lock generated
View File

@@ -130,11 +130,11 @@
},
"den": {
"locked": {
"lastModified": 1774303016,
"narHash": "sha256-uv/Mzsh80XNtvZWaDDbo33w6M18CPLLPp+3bF1tn9Cc=",
"lastModified": 1774498900,
"narHash": "sha256-THw/ly8KvXGQ0EI+Nhu/Eo9w8w7wtgHhAeWpteNiz/Q=",
"owner": "vic",
"repo": "den",
"rev": "ece9f9ec1647e82c96e4c21bb9c7060678e21d43",
"rev": "eb92bbfdefd22b76fa5781e8adbeff42c4fe429e",
"type": "github"
},
"original": {
@@ -191,11 +191,11 @@
"rust-analyzer-src": "rust-analyzer-src"
},
"locked": {
"lastModified": 1774336807,
"narHash": "sha256-9Wu6fqKbjHBh//Rlx0p/jFVobZwgvBlBov2W2H1kfAI=",
"lastModified": 1774510107,
"narHash": "sha256-UDNmYHtwG73aNs48ng7x0i3BWv6lkSn5KHxkNi1ivbU=",
"owner": "nix-community",
"repo": "fenix",
"rev": "2b5c85b86be24ff24e9586a56101e22f90ead9d4",
"rev": "620b63b95c258486d2d381bc174fa7e73adf6462",
"type": "github"
},
"original": {
@@ -414,6 +414,24 @@
"type": "github"
}
},
"flake-utils_3": {
"inputs": {
"systems": "systems_6"
},
"locked": {
"lastModified": 1731533236,
"narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "flake-utils",
"type": "github"
}
},
"himalaya": {
"inputs": {
"fenix": "fenix_2",
@@ -441,11 +459,11 @@
]
},
"locked": {
"lastModified": 1774293042,
"narHash": "sha256-OEBV+Y5I4Ldu98k0KvGXRfJYh+jjE8ocCSL/dxTGs1s=",
"lastModified": 1774379316,
"narHash": "sha256-0nGNxWDUH2Hzlj/R3Zf4FEK6fsFNB/dvewuboSRZqiI=",
"owner": "nix-community",
"repo": "home-manager",
"rev": "bc357c75e3142a31b849ba49c5299fb52c61cf59",
"rev": "1eb0549a1ab3fe3f5acf86668249be15fa0e64f7",
"type": "github"
},
"original": {
@@ -457,11 +475,11 @@
"homebrew-cask": {
"flake": false,
"locked": {
"lastModified": 1774337162,
"narHash": "sha256-ocx/hp4kACHjMcPKnCRk3/xZTvzm9LUQU4+OfXMZZhI=",
"lastModified": 1774511321,
"narHash": "sha256-W8rHd4rxUT7gwxLZ0i0VQbvIrosyau+H6Fa3nzXQAhA=",
"owner": "homebrew",
"repo": "homebrew-cask",
"rev": "15d634e0a2c56c85312ad79f544c847d3080ea2a",
"rev": "58f83e03c227201b3eaa28a1f0413eb3cd8cbd98",
"type": "github"
},
"original": {
@@ -473,11 +491,11 @@
"homebrew-core": {
"flake": false,
"locked": {
"lastModified": 1774336831,
"narHash": "sha256-QHI9Py0ZW81Nik/TUzzgCd30DBPVFUz/B2Im4wz1n2Q=",
"lastModified": 1774507765,
"narHash": "sha256-YnUOneN3De1NdT0BhpZoLr1ROfphMQQ2ogpgG8ThfdQ=",
"owner": "homebrew",
"repo": "homebrew-core",
"rev": "3d604bbb18b384e64fa759c7c7778a9436c7d4d5",
"rev": "dc21b19f04878fb04b9ed3a3fa1deb203162ece7",
"type": "github"
},
"original": {
@@ -593,11 +611,11 @@
"treefmt-nix": "treefmt-nix"
},
"locked": {
"lastModified": 1774333104,
"narHash": "sha256-acWWrX21golBLxEQ+RxEow2MZ7KxFi8iCH1LUXUL4eg=",
"lastModified": 1774495900,
"narHash": "sha256-3nR7HKulLSib37PWcWrfELuSrikFLiTqAqX2HQ9dV7g=",
"owner": "numtide",
"repo": "llm-agents.nix",
"rev": "2935ee5defa6159e56cc31ee122f8caa3772c174",
"rev": "3e06fd5f99381f8101c8e7b5a1473154dd0095cd",
"type": "github"
},
"original": {
@@ -637,11 +655,11 @@
]
},
"locked": {
"lastModified": 1774310655,
"narHash": "sha256-wKLJ5XMA1x3rcJOckqp0VC6QkJ9IKJwANG37VMbYxvw=",
"lastModified": 1774483626,
"narHash": "sha256-8VAX9GXNfv4eBj0qBEf/Rc2/E6G0SBEpuo2A5plw34I=",
"owner": "nix-community",
"repo": "neovim-nightly-overlay",
"rev": "0fcef258e524f08cdcaaa163c37f14b892f6fd88",
"rev": "5deaa19e80e1c0695f7fa8a16e13a704fd08f96e",
"type": "github"
},
"original": {
@@ -653,11 +671,11 @@
"neovim-src": {
"flake": false,
"locked": {
"lastModified": 1774309203,
"narHash": "sha256-iCxl6+jjAXSLmmoYjbgO6k2/hm/a2EVxMW44oCA1GWs=",
"lastModified": 1774472446,
"narHash": "sha256-Hp4A0llEmBvvNuw5uKOz+BA86X7TmXZ1vUK0StiMdVs=",
"owner": "neovim",
"repo": "neovim",
"rev": "1de1c08210076edbdc23fba86d9ffff25b54cbd8",
"rev": "c9e961994b16ed841be43541ef550bf3d3f043ec",
"type": "github"
},
"original": {
@@ -750,11 +768,11 @@
},
"nixpkgs_5": {
"locked": {
"lastModified": 1774337345,
"narHash": "sha256-1Bm/n42nuCHBEmSsp/mOAFLHvukAc2pyQgSDzIJ6nfY=",
"lastModified": 1774511072,
"narHash": "sha256-I3/ioUvXgXppizwI2rfmcZU8w7PHmU2tNz27tZTOib8=",
"owner": "nixos",
"repo": "nixpkgs",
"rev": "ebebb7072e01d84afec27c5b4581f43041c9f36d",
"rev": "11f32dd3f0be2df8c88fc539c627805bf3a9ff26",
"type": "github"
},
"original": {
@@ -781,6 +799,22 @@
}
},
"nixpkgs_7": {
"locked": {
"lastModified": 1769188852,
"narHash": "sha256-aBAGyMum27K7cP5OR7BMioJOF3icquJMZDDgk6ZEg1A=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "a1bab9e494f5f4939442a57a58d0449a109593fe",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixpkgs-unstable",
"repo": "nixpkgs",
"type": "github"
}
},
"nixpkgs_8": {
"locked": {
"lastModified": 1765934234,
"narHash": "sha256-pJjWUzNnjbIAMIc5gRFUuKCDQ9S1cuh3b2hKgA7Mc4A=",
@@ -819,11 +853,11 @@
"pi-agent-stuff": {
"flake": false,
"locked": {
"lastModified": 1774288055,
"narHash": "sha256-VZFa3PCOYM3Iz8b+TXPTnQDHNpRK5wvkateuQz0lu0E=",
"lastModified": 1774394773,
"narHash": "sha256-HguiVoKS87LEnqdUPLXv6VNDlA4zg9+QImZ3YnhlR2c=",
"owner": "mitsuhiko",
"repo": "agent-stuff",
"rev": "7ca2deb75f4a1853bad7d93416c72838080c5e55",
"rev": "3bf6bd34e516af81d9c2b313b568031a785a15e2",
"type": "github"
},
"original": {
@@ -851,11 +885,11 @@
"pi-harness": {
"flake": false,
"locked": {
"lastModified": 1774276819,
"narHash": "sha256-BrbgnFFd4GHJhKa0MuaT2o/LvwLf4GXPbJi6j3H9AHk=",
"lastModified": 1774378950,
"narHash": "sha256-TATerZDCk4mHCYxTf11jKPW9yPVvUDQP40RyvvoE5c8=",
"owner": "aliou",
"repo": "pi-harness",
"rev": "5bbf0cd6d6ee35d978ca32a070d9d53fbbfe2571",
"rev": "9a55e78b0650f63d368208d16e0d0fd7ba1e64a3",
"type": "github"
},
"original": {
@@ -913,6 +947,25 @@
"type": "github"
}
},
"qmd": {
"inputs": {
"flake-utils": "flake-utils_2",
"nixpkgs": "nixpkgs_7"
},
"locked": {
"lastModified": 1773490238,
"narHash": "sha256-13jhDU2wt4/SNR8oblwP6URr4SEVgMS6roiOpQEWQiY=",
"owner": "tobi",
"repo": "qmd",
"rev": "2b8f329d7e4419af736a50e917057f685ad41110",
"type": "github"
},
"original": {
"owner": "tobi",
"repo": "qmd",
"type": "github"
}
},
"root": {
"inputs": {
"code-review-nvim": "code-review-nvim",
@@ -947,6 +1000,7 @@
"pi-harness": "pi-harness",
"pi-mcp-adapter": "pi-mcp-adapter",
"pi-rose-pine": "pi-rose-pine",
"qmd": "qmd",
"sops-nix": "sops-nix",
"zjstatus": "zjstatus"
}
@@ -954,11 +1008,11 @@
"rust-analyzer-src": {
"flake": false,
"locked": {
"lastModified": 1774276450,
"narHash": "sha256-B+2C4/9l+ggImtYBwbzEDY3v9DFcDS7hNGDUI+E460o=",
"lastModified": 1774454876,
"narHash": "sha256-bwkM8HseUs/22x+hy6FWvJMP6q/2CKBrm4sYxz9rMY8=",
"owner": "rust-lang",
"repo": "rust-analyzer",
"rev": "da2fe7fed8960a4e351818d52bf444d7c9569da7",
"rev": "9253d39eab8b9c9da3c1412fc94764e01d55a02b",
"type": "github"
},
"original": {
@@ -1118,6 +1172,21 @@
"type": "github"
}
},
"systems_6": {
"locked": {
"lastModified": 1681028828,
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
"owner": "nix-systems",
"repo": "default",
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
"type": "github"
},
"original": {
"owner": "nix-systems",
"repo": "default",
"type": "github"
}
},
"treefmt-nix": {
"inputs": {
"nixpkgs": [
@@ -1160,8 +1229,8 @@
"zjstatus": {
"inputs": {
"crane": "crane",
"flake-utils": "flake-utils_2",
"nixpkgs": "nixpkgs_7",
"flake-utils": "flake-utils_3",
"nixpkgs": "nixpkgs_8",
"rust-overlay": "rust-overlay"
},
"locked": {

View File

@@ -88,6 +88,7 @@
url = "github:zenobi-us/pi-rose-pine";
flake = false;
};
qmd.url = "github:tobi/qmd";
sops-nix = {
url = "github:Mic92/sops-nix";
inputs.nixpkgs.follows = "nixpkgs";

View File

@@ -12,12 +12,16 @@
- Use Nushell (`nu`) for scripting.
- Do not use Python, Perl, Lua, awk, or any other scripting language. You are programatically blocked from doing so.
## Workflow
- Always complete the requested work.
- If there is any ambiguity about what to do next, do NOT make a decision yourself. Stop your work and ask.
- Do not end with “If you want me to…” or “I can…”; take the next necessary step and finish the job without waiting for additional confirmation.
- Do not future-proof things. Stick to the original plan.
- Do not add fallbacks or backward compatibility unless explicitly required by the user. By default, replace the previous implementation with the new one entirely.
## Validation
- Do not ignore failing tests or checks, even if they appear unrelated to your changes.
- After completing and validating your work, the final step is to run the project's full validation and test commands and ensure they all pass.
## Workflow
- Always complete the requested work.
- Do not end with “If you want me to…” or “I can…”; take the next necessary step and finish the job without waiting for additional confirmation.

View File

@@ -0,0 +1,687 @@
import { readFile, writeFile, mkdir, readdir } from "node:fs/promises";
import * as fs from "node:fs";
import * as os from "node:os";
import * as path from "node:path";
import * as crypto from "node:crypto";
import { Box, Text } from "@mariozechner/pi-tui";
import type { ExtensionAPI, ExtensionContext, ExtensionCommandContext, Model } from "@mariozechner/pi-coding-agent";
import {
createAgentSession,
DefaultResourceLoader,
getAgentDir,
SessionManager,
SettingsManager,
} from "@mariozechner/pi-coding-agent";
interface IngestManifest {
version: number;
job_id: string;
note_id: string;
operation: string;
requested_at: string;
title: string;
source_relpath: string;
source_path: string;
input_path: string;
archive_path: string;
output_path: string;
transcript_path: string;
result_path: string;
session_dir: string;
source_hash: string;
last_generated_output_hash?: string | null;
force_overwrite_generated?: boolean;
source_transport?: string;
}
interface IngestResult {
success: boolean;
job_id: string;
note_id: string;
archive_path: string;
source_hash: string;
session_dir: string;
output_path?: string;
output_hash?: string;
conflict_path?: string;
write_mode?: "create" | "overwrite" | "force-overwrite" | "conflict";
updated_main_output?: boolean;
transcript_path?: string;
error?: string;
}
interface FrontmatterInfo {
values: Record<string, string>;
body: string;
}
interface RenderedPage {
path: string;
image: {
type: "image";
source: {
type: "base64";
mediaType: string;
data: string;
};
};
}
const TRANSCRIBE_SKILL = "notability-transcribe";
const NORMALIZE_SKILL = "notability-normalize";
const STATUS_TYPE = "notability-status";
const DEFAULT_TRANSCRIBE_THINKING = "low" as const;
const DEFAULT_NORMALIZE_THINKING = "off" as const;
const PREFERRED_VISION_MODEL: [string, string] = ["openai-codex", "gpt-5.4"];
function getNotesRoot(): string {
return process.env.NOTABILITY_NOTES_DIR ?? path.join(os.homedir(), "Notes");
}
function getDataRoot(): string {
return process.env.NOTABILITY_DATA_ROOT ?? path.join(os.homedir(), ".local", "share", "notability-ingest");
}
function getRenderRoot(): string {
return process.env.NOTABILITY_RENDER_ROOT ?? path.join(getDataRoot(), "rendered-pages");
}
function getNotabilityScriptDir(): string {
return path.join(getAgentDir(), "notability");
}
function getSkillPath(skillName: string): string {
return path.join(getAgentDir(), "skills", skillName, "SKILL.md");
}
function stripFrontmatterBlock(text: string): string {
const trimmed = text.trim();
if (!trimmed.startsWith("---\n")) return trimmed;
const end = trimmed.indexOf("\n---\n", 4);
if (end === -1) return trimmed;
return trimmed.slice(end + 5).trim();
}
function stripCodeFence(text: string): string {
const trimmed = text.trim();
const match = trimmed.match(/^```(?:markdown|md)?\n([\s\S]*?)\n```$/i);
return match ? match[1].trim() : trimmed;
}
function parseFrontmatter(text: string): FrontmatterInfo {
const trimmed = stripCodeFence(text);
if (!trimmed.startsWith("---\n")) {
return { values: {}, body: trimmed };
}
const end = trimmed.indexOf("\n---\n", 4);
if (end === -1) {
return { values: {}, body: trimmed };
}
const block = trimmed.slice(4, end);
const body = trimmed.slice(end + 5).trim();
const values: Record<string, string> = {};
for (const line of block.split("\n")) {
const idx = line.indexOf(":");
if (idx === -1) continue;
const key = line.slice(0, idx).trim();
const value = line.slice(idx + 1).trim();
values[key] = value;
}
return { values, body };
}
function quoteYaml(value: string): string {
return JSON.stringify(value);
}
function sha256(content: string | Buffer): string {
return crypto.createHash("sha256").update(content).digest("hex");
}
async function sha256File(filePath: string): Promise<string> {
const buffer = await readFile(filePath);
return sha256(buffer);
}
function extractTitle(normalized: string, fallbackTitle: string): string {
const parsed = parseFrontmatter(normalized);
const frontmatterTitle = parsed.values.title?.replace(/^['"]|['"]$/g, "").trim();
if (frontmatterTitle) return frontmatterTitle;
const heading = parsed.body
.split("\n")
.map((line) => line.trim())
.find((line) => line.startsWith("# "));
if (heading) return heading.replace(/^#\s+/, "").trim();
return fallbackTitle;
}
function sourceFormat(filePath: string): string {
const extension = path.extname(filePath).toLowerCase();
if (extension === ".pdf") return "pdf";
if (extension === ".png") return "png";
return extension.replace(/^\./, "") || "unknown";
}
function buildMarkdown(manifest: IngestManifest, normalized: string): string {
const parsed = parseFrontmatter(normalized);
const title = extractTitle(normalized, manifest.title);
const now = new Date().toISOString().replace(/\.\d{3}Z$/, "Z");
const created = manifest.requested_at.slice(0, 10);
const body = parsed.body.trim();
const outputBody = body.length > 0 ? body : `# ${title}\n`;
return [
"---",
`title: ${quoteYaml(title)}`,
`created: ${quoteYaml(created)}`,
`updated: ${quoteYaml(now.slice(0, 10))}`,
`source: ${quoteYaml("notability")}`,
`source_transport: ${quoteYaml(manifest.source_transport ?? "webdav")}`,
`source_relpath: ${quoteYaml(manifest.source_relpath)}`,
`note_id: ${quoteYaml(manifest.note_id)}`,
`managed_by: ${quoteYaml("notability-ingest")}`,
`source_file: ${quoteYaml(manifest.archive_path)}`,
`source_file_hash: ${quoteYaml(`sha256:${manifest.source_hash}`)}`,
`source_format: ${quoteYaml(sourceFormat(manifest.archive_path))}`,
`status: ${quoteYaml("active")}`,
"tags:",
" - handwritten",
" - notability",
"---",
"",
outputBody,
"",
].join("\n");
}
function conflictPathFor(outputPath: string): string {
const parsed = path.parse(outputPath);
const stamp = new Date().toISOString().replace(/[:]/g, "-").replace(/\.\d{3}Z$/, "Z");
return path.join(parsed.dir, `${parsed.name}.conflict-${stamp}${parsed.ext}`);
}
async function ensureParent(filePath: string): Promise<void> {
await mkdir(path.dirname(filePath), { recursive: true });
}
async function loadSkillText(skillName: string): Promise<string> {
const raw = await readFile(getSkillPath(skillName), "utf8");
return stripFrontmatterBlock(raw).trim();
}
function normalizePathArg(arg: string): string {
return arg.startsWith("@") ? arg.slice(1) : arg;
}
function resolveModel(ctx: ExtensionContext, requireImage = false): Model {
const available = ctx.modelRegistry.getAvailable();
const matching = requireImage ? available.filter((model) => model.input.includes("image")) : available;
if (matching.length === 0) {
throw new Error(
requireImage
? "No image-capable model configured for pi note ingestion"
: "No available model configured for pi note ingestion",
);
}
if (ctx.model && (!requireImage || ctx.model.input.includes("image"))) {
if (!requireImage) return ctx.model;
}
if (requireImage) {
const [provider, id] = PREFERRED_VISION_MODEL;
const preferred = matching.find((model) => model.provider === provider && model.id === id);
if (preferred) return preferred;
const subscriptionModel = matching.find(
(model) => model.provider !== "opencode" && model.provider !== "opencode-go",
);
if (subscriptionModel) return subscriptionModel;
}
if (ctx.model && (!requireImage || ctx.model.input.includes("image"))) {
return ctx.model;
}
return matching[0];
}
async function runSkillPrompt(
ctx: ExtensionContext,
systemPrompt: string,
prompt: string,
images: RenderedPage[] = [],
thinkingLevel: "off" | "low" = "off",
): Promise<string> {
if (images.length > 0) {
const model = resolveModel(ctx, true);
const { execFile } = await import("node:child_process");
const promptPath = path.join(os.tmpdir(), `pi-note-ingest-${crypto.randomUUID()}.md`);
await writeFile(promptPath, `${prompt}\n`);
const args = [
"45s",
"pi",
"--model",
`${model.provider}/${model.id}`,
"--thinking",
thinkingLevel,
"--no-tools",
"--no-session",
"-p",
...images.map((page) => `@${page.path}`),
`@${promptPath}`,
];
try {
const output = await new Promise<string>((resolve, reject) => {
execFile("timeout", args, { cwd: ctx.cwd, env: process.env, maxBuffer: 10 * 1024 * 1024 }, (error, stdout, stderr) => {
if ((stdout ?? "").trim().length > 0) {
resolve(stdout);
return;
}
if (error) {
reject(new Error(stderr || stdout || error.message));
return;
}
resolve(stdout);
});
});
return stripCodeFence(output).trim();
} finally {
try {
fs.unlinkSync(promptPath);
} catch {
// Ignore temp file cleanup failures.
}
}
}
const agentDir = getAgentDir();
const settingsManager = SettingsManager.create(ctx.cwd, agentDir);
const resourceLoader = new DefaultResourceLoader({
cwd: ctx.cwd,
agentDir,
settingsManager,
noExtensions: true,
noPromptTemplates: true,
noThemes: true,
noSkills: true,
systemPromptOverride: () => systemPrompt,
appendSystemPromptOverride: () => [],
agentsFilesOverride: () => ({ agentsFiles: [] }),
});
await resourceLoader.reload();
const { session } = await createAgentSession({
model: resolveModel(ctx, images.length > 0),
thinkingLevel,
sessionManager: SessionManager.inMemory(),
modelRegistry: ctx.modelRegistry,
resourceLoader,
tools: [],
});
let output = "";
const unsubscribe = session.subscribe((event) => {
if (event.type === "message_update" && event.assistantMessageEvent.type === "text_delta") {
output += event.assistantMessageEvent.delta;
}
});
try {
await session.prompt(prompt, {
images: images.map((page) => page.image),
});
} finally {
unsubscribe();
}
if (!output.trim()) {
const assistantMessages = session.messages.filter((message) => message.role === "assistant");
const lastAssistant = assistantMessages.at(-1);
if (lastAssistant && Array.isArray(lastAssistant.content)) {
output = lastAssistant.content
.filter((part) => part.type === "text")
.map((part) => part.text)
.join("");
}
}
session.dispose();
return stripCodeFence(output).trim();
}
async function renderPdfPages(pdfPath: string, jobId: string): Promise<RenderedPage[]> {
const renderDir = path.join(getRenderRoot(), jobId);
await mkdir(renderDir, { recursive: true });
const prefix = path.join(renderDir, "page");
const args = ["-png", "-r", "200", pdfPath, prefix];
const { execFile } = await import("node:child_process");
await new Promise<void>((resolve, reject) => {
execFile("pdftoppm", args, (error) => {
if (error) reject(error);
else resolve();
});
});
const entries = await readdir(renderDir);
const pngs = entries
.filter((entry) => entry.endsWith(".png"))
.sort((left, right) => left.localeCompare(right, undefined, { numeric: true }));
if (pngs.length === 0) {
throw new Error(`No rendered pages produced for ${pdfPath}`);
}
const pages: RenderedPage[] = [];
for (const entry of pngs) {
const pagePath = path.join(renderDir, entry);
const buffer = await readFile(pagePath);
pages.push({
path: pagePath,
image: {
type: "image",
source: {
type: "base64",
mediaType: "image/png",
data: buffer.toString("base64"),
},
},
});
}
return pages;
}
async function loadImagePage(imagePath: string): Promise<RenderedPage> {
const extension = path.extname(imagePath).toLowerCase();
const mediaType = extension === ".png" ? "image/png" : undefined;
if (!mediaType) {
throw new Error(`Unsupported image input format for ${imagePath}`);
}
const buffer = await readFile(imagePath);
return {
path: imagePath,
image: {
type: "image",
source: {
type: "base64",
mediaType,
data: buffer.toString("base64"),
},
},
};
}
async function renderInputPages(inputPath: string, jobId: string): Promise<RenderedPage[]> {
const extension = path.extname(inputPath).toLowerCase();
if (extension === ".pdf") {
return await renderPdfPages(inputPath, jobId);
}
if (extension === ".png") {
return [await loadImagePage(inputPath)];
}
throw new Error(`Unsupported Notability input format: ${inputPath}`);
}
async function findManagedOutputs(noteId: string): Promise<string[]> {
const matches: string[] = [];
const stack = [getNotesRoot()];
while (stack.length > 0) {
const currentDir = stack.pop();
if (!currentDir || !fs.existsSync(currentDir)) continue;
const entries = await readdir(currentDir, { withFileTypes: true });
for (const entry of entries) {
if (entry.name.startsWith(".")) continue;
const fullPath = path.join(currentDir, entry.name);
if (entry.isDirectory()) {
stack.push(fullPath);
continue;
}
if (!entry.isFile() || !entry.name.endsWith(".md")) continue;
try {
const parsed = parseFrontmatter(await readFile(fullPath, "utf8"));
const managedBy = parsed.values.managed_by?.replace(/^['"]|['"]$/g, "");
const frontmatterNoteId = parsed.values.note_id?.replace(/^['"]|['"]$/g, "");
if (managedBy === "notability-ingest" && frontmatterNoteId === noteId) {
matches.push(fullPath);
}
} catch {
// Ignore unreadable or malformed files while scanning the notebook.
}
}
}
return matches.sort();
}
async function resolveManagedOutputPath(noteId: string, configuredOutputPath: string): Promise<string> {
if (fs.existsSync(configuredOutputPath)) {
const parsed = parseFrontmatter(await readFile(configuredOutputPath, "utf8"));
const managedBy = parsed.values.managed_by?.replace(/^['"]|['"]$/g, "");
const frontmatterNoteId = parsed.values.note_id?.replace(/^['"]|['"]$/g, "");
if (managedBy === "notability-ingest" && frontmatterNoteId === noteId) {
return configuredOutputPath;
}
}
const discovered = await findManagedOutputs(noteId);
if (discovered.length === 0) return configuredOutputPath;
if (discovered.length === 1) return discovered[0];
throw new Error(
`Multiple managed note files found for ${noteId}: ${discovered.join(", ")}`,
);
}
async function determineWriteTarget(manifest: IngestManifest, markdown: string): Promise<{
outputPath: string;
writePath: string;
writeMode: "create" | "overwrite" | "force-overwrite" | "conflict";
updatedMainOutput: boolean;
}> {
const outputPath = await resolveManagedOutputPath(manifest.note_id, manifest.output_path);
if (!fs.existsSync(outputPath)) {
return { outputPath, writePath: outputPath, writeMode: "create", updatedMainOutput: true };
}
const existing = await readFile(outputPath, "utf8");
const existingHash = sha256(existing);
const parsed = parseFrontmatter(existing);
const isManaged = parsed.values.managed_by?.replace(/^['"]|['"]$/g, "") === "notability-ingest";
const sameNoteId = parsed.values.note_id?.replace(/^['"]|['"]$/g, "") === manifest.note_id;
if (manifest.last_generated_output_hash && existingHash === manifest.last_generated_output_hash) {
return { outputPath, writePath: outputPath, writeMode: "overwrite", updatedMainOutput: true };
}
if (manifest.force_overwrite_generated && isManaged && sameNoteId) {
return { outputPath, writePath: outputPath, writeMode: "force-overwrite", updatedMainOutput: true };
}
return {
outputPath,
writePath: conflictPathFor(outputPath),
writeMode: "conflict",
updatedMainOutput: false,
};
}
async function writeIngestResult(resultPath: string, payload: IngestResult): Promise<void> {
await ensureParent(resultPath);
await writeFile(resultPath, JSON.stringify(payload, null, 2));
}
async function ingestManifest(manifestPath: string, ctx: ExtensionContext): Promise<IngestResult> {
const manifest = JSON.parse(await readFile(manifestPath, "utf8")) as IngestManifest;
await ensureParent(manifest.transcript_path);
await ensureParent(manifest.result_path);
await mkdir(manifest.session_dir, { recursive: true });
const normalizeSkill = await loadSkillText(NORMALIZE_SKILL);
const pages = await renderInputPages(manifest.input_path, manifest.job_id);
const pageSummary = pages.map((page, index) => `- page ${index + 1}: ${page.path}`).join("\n");
const transcriptPrompt = [
"Transcribe this note into clean Markdown.",
"Read it like a human and preserve the intended reading order and visible structure.",
"Keep headings, lists, and paragraphs when they are visible.",
"Do not summarize. Do not add commentary. Return Markdown only.",
"Rendered pages:",
pageSummary,
].join("\n\n");
let transcript = await runSkillPrompt(
ctx,
"",
transcriptPrompt,
pages,
DEFAULT_TRANSCRIBE_THINKING,
);
if (!transcript.trim()) {
throw new Error("Transcription skill returned empty output");
}
await writeFile(manifest.transcript_path, `${transcript.trim()}\n`);
const normalizePrompt = [
`Note ID: ${manifest.note_id}`,
`Source path: ${manifest.source_relpath}`,
`Preferred output path: ${manifest.output_path}`,
"Normalize the following transcription into clean Markdown.",
"Restore natural prose formatting and intended reading order when the transcription contains OCR or layout artifacts.",
"If words are split across separate lines but clearly belong to the same phrase or sentence, merge them.",
"Return only Markdown. No code fences.",
"",
"<transcription>",
transcript.trim(),
"</transcription>",
].join("\n");
const normalized = await runSkillPrompt(
ctx,
normalizeSkill,
normalizePrompt,
[],
DEFAULT_NORMALIZE_THINKING,
);
if (!normalized.trim()) {
throw new Error("Normalization skill returned empty output");
}
const markdown = buildMarkdown(manifest, normalized);
const target = await determineWriteTarget(manifest, markdown);
await ensureParent(target.writePath);
await writeFile(target.writePath, markdown);
const result: IngestResult = {
success: true,
job_id: manifest.job_id,
note_id: manifest.note_id,
archive_path: manifest.archive_path,
source_hash: manifest.source_hash,
session_dir: manifest.session_dir,
output_path: target.outputPath,
output_hash: target.updatedMainOutput ? await sha256File(target.writePath) : undefined,
conflict_path: target.writeMode === "conflict" ? target.writePath : undefined,
write_mode: target.writeMode,
updated_main_output: target.updatedMainOutput,
transcript_path: manifest.transcript_path,
};
await writeIngestResult(manifest.result_path, result);
return result;
}
async function runScript(scriptName: string, args: string[]): Promise<string> {
const { execFile } = await import("node:child_process");
const scriptPath = path.join(getNotabilityScriptDir(), scriptName);
return await new Promise<string>((resolve, reject) => {
execFile("nu", [scriptPath, ...args], (error, stdout, stderr) => {
if (error) {
reject(new Error(stderr || stdout || error.message));
return;
}
resolve(stdout.trim());
});
});
}
function splitArgs(input: string): string[] {
return input
.trim()
.split(/\s+/)
.filter((part) => part.length > 0);
}
function postStatus(pi: ExtensionAPI, content: string): void {
pi.sendMessage({
customType: STATUS_TYPE,
content,
display: true,
});
}
export default function noteIngestExtension(pi: ExtensionAPI) {
pi.registerMessageRenderer(STATUS_TYPE, (message, _options, theme) => {
const box = new Box(1, 1, (text) => theme.bg("customMessageBg", text));
box.addChild(new Text(message.content, 0, 0));
return box;
});
pi.registerCommand("note-status", {
description: "Show Notability ingest status",
handler: async (args, _ctx) => {
const output = await runScript("status.nu", splitArgs(args));
postStatus(pi, output.length > 0 ? output : "No status output");
},
});
pi.registerCommand("note-reingest", {
description: "Enqueue a note for reingestion",
handler: async (args, _ctx) => {
const trimmed = args.trim();
if (!trimmed) {
postStatus(pi, "Usage: /note-reingest <note-id> [--latest-source|--latest-archive] [--force-overwrite-generated]");
return;
}
const output = await runScript("reingest.nu", splitArgs(trimmed));
postStatus(pi, output.length > 0 ? output : "Reingest enqueued");
},
});
pi.registerCommand("note-ingest", {
description: "Ingest a queued Notability job manifest",
handler: async (args, ctx: ExtensionCommandContext) => {
const manifestPath = normalizePathArg(args.trim());
if (!manifestPath) {
throw new Error("Usage: /note-ingest <job.json>");
}
let resultPath = "";
try {
const raw = await readFile(manifestPath, "utf8");
const manifest = JSON.parse(raw) as IngestManifest;
resultPath = manifest.result_path;
const result = await ingestManifest(manifestPath, ctx);
postStatus(pi, `Ingested ${result.note_id} (${result.write_mode})`);
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
if (resultPath) {
const manifest = JSON.parse(await readFile(manifestPath, "utf8")) as IngestManifest;
await writeIngestResult(resultPath, {
success: false,
job_id: manifest.job_id,
note_id: manifest.note_id,
archive_path: manifest.archive_path,
source_hash: manifest.source_hash,
session_dir: manifest.session_dir,
error: message,
});
}
throw error;
}
},
});
}

View File

@@ -359,7 +359,7 @@ type BookmarkRef = {
type ReviewTarget =
| { type: "workingCopy" }
| { type: "baseBookmark"; bookmark: string; remote?: string }
| { type: "change"; sha: string; title?: string }
| { type: "change"; changeId: string; title?: string }
| { type: "pullRequest"; prNumber: number; baseBookmark: string; baseRemote?: string; title: string }
| { type: "folder"; paths: string[] };
@@ -377,9 +377,9 @@ const BASE_BOOKMARK_PROMPT_FALLBACK =
"Review the code changes against the base bookmark '{bookmark}'. Start by finding the merge-base revision between the working copy and {bookmark}, then run `jj diff --from <merge-base> --to @` to see what changes would land on the {bookmark} bookmark. Provide prioritized, actionable findings.";
const CHANGE_PROMPT_WITH_TITLE =
'Review the code changes introduced by change {sha} ("{title}"). Provide prioritized, actionable findings.';
'Review the code changes introduced by change {changeId} ("{title}"). Provide prioritized, actionable findings.';
const CHANGE_PROMPT = "Review the code changes introduced by change {sha}. Provide prioritized, actionable findings.";
const CHANGE_PROMPT = "Review the code changes introduced by change {changeId}. Provide prioritized, actionable findings.";
const PULL_REQUEST_PROMPT =
'Review pull request #{prNumber} ("{title}") against the base bookmark \'{baseBookmark}\'. The merge-base revision for this comparison is {mergeBaseSha}. Run `jj diff --from {mergeBaseSha} --to @` to inspect the changes that would be merged. Provide prioritized, actionable findings.';
@@ -748,22 +748,22 @@ async function getMergeBase(
/**
* Get list of recent changes
*/
async function getRecentChanges(pi: ExtensionAPI, limit: number = 10): Promise<Array<{ sha: string; title: string }>> {
async function getRecentChanges(pi: ExtensionAPI, limit: number = 10): Promise<Array<{ changeId: string; title: string }>> {
const { stdout, code } = await pi.exec("jj", [
"log",
"-n",
`${limit}`,
"--no-graph",
"-T",
'commit_id ++ "\\t" ++ description.first_line() ++ "\\n"',
'change_id.shortest(8) ++ "\\t" ++ description.first_line() ++ "\\n"',
]);
if (code !== 0) return [];
return parseNonEmptyLines(stdout)
.filter((line) => line.trim())
.map((line) => {
const [sha, ...rest] = line.trim().split("\t");
return { sha, title: rest.join(" ") };
const [changeId, ...rest] = line.trim().split("\t");
return { changeId, title: rest.join(" ") };
});
}
@@ -979,9 +979,9 @@ async function buildReviewPrompt(
case "change":
if (target.title) {
return CHANGE_PROMPT_WITH_TITLE.replace("{sha}", target.sha).replace("{title}", target.title);
return CHANGE_PROMPT_WITH_TITLE.replace("{changeId}", target.changeId).replace("{title}", target.title);
}
return CHANGE_PROMPT.replace("{sha}", target.sha);
return CHANGE_PROMPT.replace("{changeId}", target.changeId);
case "pullRequest": {
const baseBookmarkLabel = bookmarkRefToLabel({ name: target.baseBookmark, remote: target.baseRemote });
@@ -1014,8 +1014,7 @@ function getUserFacingHint(target: ReviewTarget): string {
case "baseBookmark":
return `changes against '${bookmarkRefToLabel({ name: target.bookmark, remote: target.remote })}'`;
case "change": {
const shortSha = target.sha.slice(0, 7);
return target.title ? `change ${shortSha}: ${target.title}` : `change ${shortSha}`;
return target.title ? `change ${target.changeId}: ${target.title}` : `change ${target.changeId}`;
}
case "pullRequest": {
@@ -1441,12 +1440,12 @@ export default function reviewExtension(pi: ExtensionAPI) {
}
const items: SelectItem[] = changes.map((change) => ({
value: change.sha,
label: `${change.sha.slice(0, 7)} ${change.title}`,
value: change.changeId,
label: `${change.changeId} ${change.title}`,
description: "",
}));
const result = await ctx.ui.custom<{ sha: string; title: string } | null>((tui, theme, keybindings, done) => {
const result = await ctx.ui.custom<{ changeId: string; title: string } | null>((tui, theme, keybindings, done) => {
const container = new Container();
container.addChild(new DynamicBorder((str) => theme.fg("accent", str)));
container.addChild(new Text(theme.fg("accent", theme.bold("Select change to review"))));
@@ -1480,7 +1479,7 @@ export default function reviewExtension(pi: ExtensionAPI) {
});
selectList.onSelect = (item) => {
const change = changes.find((c) => c.sha === item.value);
const change = changes.find((c) => c.changeId === item.value);
if (change) {
done(change);
} else {
@@ -1532,7 +1531,7 @@ export default function reviewExtension(pi: ExtensionAPI) {
});
if (!result) return null;
return { type: "change", sha: result.sha, title: result.title };
return { type: "change", changeId: result.changeId, title: result.title };
}
@@ -1822,10 +1821,10 @@ export default function reviewExtension(pi: ExtensionAPI) {
}
case "change": {
const sha = parts[1];
if (!sha) return { target: null, extraInstruction };
const changeId = parts[1];
if (!changeId) return { target: null, extraInstruction };
const title = parts.slice(2).join(" ") || undefined;
return { target: { type: "change", sha, title }, extraInstruction };
return { target: { type: "change", changeId, title }, extraInstruction };
}

View File

@@ -3,15 +3,15 @@
"opensrc": {
"command": "npx",
"args": ["-y", "opensrc-mcp"],
"directTools": true
"lifecycle": "eager"
},
"context7": {
"url": "https://mcp.context7.com/mcp",
"directTools": true
"lifecycle": "eager"
},
"grep_app": {
"url": "https://mcp.grep.app",
"directTools": true
"lifecycle": "eager"
},
"sentry": {
"url": "https://mcp.sentry.dev/mcp",

View File

@@ -0,0 +1,36 @@
---
name: notability-normalize
description: Normalizes an exact Notability transcription into clean, searchable Markdown while preserving all original content and uncertainty markers. Use after a faithful transcription pass.
---
# Notability Normalize
You are doing a **Markdown normalization** pass on a previously transcribed Notability note.
## Rules
- Do **not** summarize.
- Do **not** remove uncertainty markers such as `[unclear: ...]`.
- Preserve all substantive content from the transcription.
- Clean up only formatting and Markdown structure.
- Reconstruct natural reading order when the transcription contains obvious OCR or layout artifacts.
- Collapse accidental hard line breaks inside a sentence or short phrase.
- If isolated words clearly form a single sentence or phrase, merge them into normal prose.
- Prefer readable Markdown headings, lists, and tables.
- Keep content in the same overall order as the transcription.
- Do not invent content.
- Do not output code fences.
- Output Markdown only.
## Output
- Produce a clean Markdown document.
- Include a top-level `#` heading if the note clearly has a title.
- Use standard Markdown lists and checkboxes.
- Represent tables as Markdown tables when practical.
- Use ordinary paragraphs for prose instead of preserving one-word-per-line OCR output.
- Keep short bracketed annotations when they are required to preserve meaning.
## Important
The source PDF remains the ground truth. When in doubt, preserve ambiguity instead of cleaning it away.

View File

@@ -0,0 +1,38 @@
---
name: notability-transcribe
description: Faithfully transcribes handwritten or mixed handwritten/typed Notability note pages into Markdown without summarizing. Use when converting note page images or PDFs into an exact textual transcription.
---
# Notability Transcribe
You are doing a **faithful transcription** pass for handwritten Notability notes.
## Rules
- Preserve the original order of content.
- Reconstruct the intended reading order from the page layout.
- Read the page in the order a human would: top-to-bottom and left-to-right, while respecting obvious grouping.
- Do **not** summarize, explain, clean up, or reorganize beyond what is necessary to transcribe faithfully.
- Preserve headings, bullets, numbered items, checkboxes, tables, separators, callouts, and obvious layout structure.
- Do **not** preserve accidental OCR-style hard line breaks when the note is clearly continuous prose or a single phrase.
- If words are staggered on the page but clearly belong to the same sentence, combine them into normal lines.
- If text is uncertain, keep the uncertainty inline as `[unclear: ...]`.
- If a word is partially legible, include the best reading and uncertainty marker.
- If there is a drawing or diagram that cannot be represented exactly, describe it minimally in brackets, for example `[diagram: arrow from A to B]`.
- Preserve language exactly as written.
- Do not invent missing words.
- Do not output code fences.
- Output Markdown only.
## Output shape
- Use headings when headings are clearly present.
- Use `- [ ]` or `- [x]` for checkboxes when visible.
- Use bullet lists for bullet lists.
- Use normal paragraphs or single-line phrases for continuous prose instead of one word per line.
- Keep side notes in the position that best preserves reading order.
- Insert blank lines between major sections.
## Safety
If a page is partly unreadable, still transcribe everything you can and mark uncertain content with `[unclear: ...]`.

View File

@@ -1,15 +1,15 @@
{inputs, ...}: final: prev: let
version = "0.22.1";
version = "0.24.0";
srcs = {
x86_64-linux =
prev.fetchurl {
url = "https://github.com/trycog/cog-cli/releases/download/v${version}/cog-linux-x86_64.tar.gz";
hash = "sha256-ET+sNXisUrHShR1gxqdumegXycXcxGzJcQOdTr5005w=";
hash = "sha256-9Ka7rPIlWtLVxRg9yNQCNz16AE4j0zGf2TW7xBXrksM=";
};
aarch64-darwin =
prev.fetchurl {
url = "https://github.com/trycog/cog-cli/releases/download/v${version}/cog-darwin-arm64.tar.gz";
hash = "sha256-jcN+DtOqr3or5C71jp7AIAz0wh73FYybCC4FRBykKO4=";
hash = "sha256-YNONHRmPGDhJeF+7rcWmrjqktYpi4b6bLl+M7IEFDtU=";
};
};
in {

View File

@@ -12,7 +12,7 @@
src = inputs.pi-harness;
pnpm = prev.pnpm_10;
fetcherVersion = 1;
hash = "sha256-FgtJnmJ0/udz2A9N2DQns+a2CspMDEDk0DPUAxmCVY4=";
hash = "sha256-dXctaspB+efF0o88AB+1JPyWg36ZT90VeagPEBHlF20=";
};
nativeBuildInputs = [

5157
modules/_overlays/qmd-package-lock.json generated Normal file

File diff suppressed because it is too large Load Diff

43
modules/_overlays/qmd.nix Normal file
View File

@@ -0,0 +1,43 @@
{inputs, ...}: final: prev: {
qmd =
prev.buildNpmPackage rec {
pname = "qmd";
version = "2.0.1";
src = inputs.qmd;
npmDepsHash = "sha256-ODpDkCQwkjqf9X5EfKmnCP4z4AjC6O/lS/zJKBs/46I=";
nativeBuildInputs = [
prev.makeWrapper
prev.python3
prev.pkg-config
prev.cmake
];
buildInputs = [prev.sqlite];
dontConfigure = true;
postPatch = ''
cp ${./qmd-package-lock.json} package-lock.json
'';
npmBuildScript = "build";
dontNpmPrune = true;
installPhase = ''
runHook preInstall
mkdir -p $out/lib/node_modules/qmd $out/bin
cp -r bin dist node_modules package.json package-lock.json LICENSE CHANGELOG.md $out/lib/node_modules/qmd/
makeWrapper ${prev.nodejs}/bin/node $out/bin/qmd \
--add-flags $out/lib/node_modules/qmd/dist/cli/qmd.js \
--set LD_LIBRARY_PATH ${prev.lib.makeLibraryPath [prev.sqlite]}
runHook postInstall
'';
meta = with prev.lib; {
description = "On-device search engine for markdown notes, meeting transcripts, and knowledge bases";
homepage = "https://github.com/tobi/qmd";
license = licenses.mit;
mainProgram = "qmd";
platforms = platforms.unix;
};
};
}

View File

@@ -31,13 +31,20 @@
};
".pi/agent/extensions/no-git.ts".source = ./_ai-tools/extensions/no-git.ts;
".pi/agent/extensions/no-scripting.ts".source = ./_ai-tools/extensions/no-scripting.ts;
".pi/agent/extensions/note-ingest.ts".source = ./_ai-tools/extensions/note-ingest.ts;
".pi/agent/extensions/review.ts".source = ./_ai-tools/extensions/review.ts;
".pi/agent/extensions/session-name.ts".source = ./_ai-tools/extensions/session-name.ts;
".pi/agent/notability" = {
source = ./hosts/_parts/tahani/notability;
recursive = true;
};
".pi/agent/skills/elixir-dev" = {
source = "${inputs.pi-elixir}/skills/elixir-dev";
recursive = true;
};
".pi/agent/skills/jujutsu/SKILL.md".source = ./_ai-tools/skills/jujutsu/SKILL.md;
".pi/agent/skills/notability-transcribe/SKILL.md".source = ./_ai-tools/skills/notability-transcribe/SKILL.md;
".pi/agent/skills/notability-normalize/SKILL.md".source = ./_ai-tools/skills/notability-normalize/SKILL.md;
".pi/agent/themes" = {
source = "${inputs.pi-rose-pine}/themes";
recursive = true;

View File

@@ -140,6 +140,11 @@
homebrew = {
enable = true;
onActivation = {
autoUpdate = true;
cleanup = "uninstall";
upgrade = true;
};
casks = [
"1password"
"alcove"

View File

@@ -74,6 +74,7 @@
url = "github:nicobailon/pi-mcp-adapter/v2.2.0";
flake = false;
};
qmd.url = "github:tobi/qmd";
# Overlay inputs
himalaya.url = "github:pimalaya/himalaya";
jj-ryu = {

View File

@@ -0,0 +1,128 @@
{
config,
inputs',
pkgs,
...
}: let
homeDir = "/home/cschmatzler";
notabilityScripts = ./notability;
dataRoot = "${homeDir}/.local/share/notability-ingest";
stateRoot = "${homeDir}/.local/state/notability-ingest";
notesRoot = "${homeDir}/Notes";
webdavRoot = "${dataRoot}/webdav-root";
userPackages = with pkgs; [
qmd
poppler-utils
rclone
sqlite
zk
];
commonPath = with pkgs;
[
inputs'.llm-agents.packages.pi
coreutils
inotify-tools
nushell
util-linux
]
++ userPackages;
commonEnvironment = {
HOME = homeDir;
NOTABILITY_ARCHIVE_ROOT = "${dataRoot}/archive";
NOTABILITY_DATA_ROOT = dataRoot;
NOTABILITY_DB_PATH = "${stateRoot}/db.sqlite";
NOTABILITY_NOTES_DIR = notesRoot;
NOTABILITY_RENDER_ROOT = "${dataRoot}/rendered-pages";
NOTABILITY_SESSIONS_ROOT = "${stateRoot}/sessions";
NOTABILITY_STATE_ROOT = stateRoot;
NOTABILITY_TRANSCRIPT_ROOT = "${stateRoot}/transcripts";
NOTABILITY_WEBDAV_ROOT = webdavRoot;
XDG_CONFIG_HOME = "${homeDir}/.config";
};
mkTmpDirRule = path: "d ${path} 0755 cschmatzler users -";
mkNotabilityService = {
description,
script,
after ? [],
requires ? [],
environment ? {},
}: {
inherit after description requires;
wantedBy = ["multi-user.target"];
path = commonPath;
environment = commonEnvironment // environment;
serviceConfig = {
ExecStart = "${pkgs.nushell}/bin/nu ${notabilityScripts}/${script}";
Group = "users";
Restart = "always";
RestartSec = 5;
User = "cschmatzler";
WorkingDirectory = homeDir;
};
};
in {
sops.secrets.tahani-notability-webdav-password = {
sopsFile = ../../../../secrets/tahani-notability-webdav-password;
format = "binary";
owner = "cschmatzler";
path = "/run/secrets/tahani-notability-webdav-password";
};
home-manager.users.cschmatzler = {
home.packages = userPackages;
home.file.".config/qmd/index.yml".text = ''
collections:
notes:
path: ${notesRoot}
pattern: "**/*.md"
'';
};
systemd.tmpfiles.rules =
builtins.map mkTmpDirRule [
notesRoot
dataRoot
webdavRoot
"${dataRoot}/archive"
"${dataRoot}/rendered-pages"
stateRoot
"${stateRoot}/jobs"
"${stateRoot}/jobs/queued"
"${stateRoot}/jobs/running"
"${stateRoot}/jobs/failed"
"${stateRoot}/jobs/done"
"${stateRoot}/jobs/results"
"${stateRoot}/sessions"
"${stateRoot}/transcripts"
];
services.caddy.virtualHosts."tahani.manticore-hippocampus.ts.net".extraConfig = ''
tls {
get_certificate tailscale
}
handle /notability* {
reverse_proxy 127.0.0.1:9980
}
'';
systemd.services.notability-webdav =
mkNotabilityService {
description = "Notability WebDAV landing zone";
script = "webdav.nu";
after = ["network.target"];
environment = {
NOTABILITY_WEBDAV_ADDR = "127.0.0.1:9980";
NOTABILITY_WEBDAV_BASEURL = "/notability";
NOTABILITY_WEBDAV_PASSWORD_FILE = config.sops.secrets.tahani-notability-webdav-password.path;
NOTABILITY_WEBDAV_USER = "notability";
};
};
systemd.services.notability-watch =
mkNotabilityService {
description = "Watch and ingest Notability WebDAV uploads";
script = "watch.nu";
after = ["notability-webdav.service"];
requires = ["notability-webdav.service"];
};
}

View File

@@ -0,0 +1,141 @@
#!/usr/bin/env nu
use ./lib.nu *
def active-job-exists [note_id: string, source_hash: string] {
let rows = (sql-json $"
select job_id
from jobs
where note_id = (sql-quote $note_id)
and source_hash = (sql-quote $source_hash)
and status != 'done'
and status != 'failed'
limit 1;
")
not ($rows | is-empty)
}
export def archive-and-version [note_id: string, source_path: path, source_relpath: string, source_size: any, source_mtime: string, source_hash: string] {
let source_size_int = ($source_size | into int)
let archive_path = (archive-path-for $note_id $source_hash $source_relpath)
cp $source_path $archive_path
let version_id = (new-version-id)
let seen_at = (now-iso)
let version_id_q = (sql-quote $version_id)
let note_id_q = (sql-quote $note_id)
let seen_at_q = (sql-quote $seen_at)
let archive_path_q = (sql-quote $archive_path)
let source_hash_q = (sql-quote $source_hash)
let source_mtime_q = (sql-quote $source_mtime)
let source_relpath_q = (sql-quote $source_relpath)
let sql = ([
"insert into versions (version_id, note_id, seen_at, archive_path, source_hash, source_size, source_mtime, source_relpath, ingest_result, session_path) values ("
$version_id_q
", "
$note_id_q
", "
$seen_at_q
", "
$archive_path_q
", "
$source_hash_q
", "
($source_size_int | into string)
", "
$source_mtime_q
", "
$source_relpath_q
", 'pending', null);"
] | str join '')
sql-run $sql | ignore
{
version_id: $version_id
seen_at: $seen_at
archive_path: $archive_path
}
}
export def enqueue-job [
note: record,
operation: string,
input_path: string,
archive_path: string,
source_hash: string,
title: string,
force_overwrite_generated: bool = false,
source_transport: string = 'webdav',
] {
if (active-job-exists $note.note_id $source_hash) {
return null
}
let job_id = (new-job-id)
let requested_at = (now-iso)
let manifest_path = (manifest-path-for $job_id 'queued')
let result_path = (result-path-for $job_id)
let transcript_path = (transcript-path-for $note.note_id $job_id)
let session_dir = ([(sessions-root) $note.note_id $job_id] | path join)
mkdir $session_dir
let manifest = {
version: 1
job_id: $job_id
note_id: $note.note_id
operation: $operation
requested_at: $requested_at
title: $title
source_relpath: $note.source_relpath
source_path: $note.source_path
input_path: $input_path
archive_path: $archive_path
output_path: $note.output_path
transcript_path: $transcript_path
result_path: $result_path
session_dir: $session_dir
source_hash: $source_hash
last_generated_output_hash: ($note.last_generated_output_hash? | default null)
force_overwrite_generated: $force_overwrite_generated
source_transport: $source_transport
}
($manifest | to json --indent 2) | save -f $manifest_path
let job_id_q = (sql-quote $job_id)
let note_id_q = (sql-quote $note.note_id)
let operation_q = (sql-quote $operation)
let requested_at_q = (sql-quote $requested_at)
let source_hash_q = (sql-quote $source_hash)
let manifest_path_q = (sql-quote $manifest_path)
let result_path_q = (sql-quote $result_path)
let sql = ([
"insert into jobs (job_id, note_id, operation, status, requested_at, source_hash, job_manifest_path, result_path) values ("
$job_id_q
", "
$note_id_q
", "
$operation_q
", 'queued', "
$requested_at_q
", "
$source_hash_q
", "
$manifest_path_q
", "
$result_path_q
");"
] | str join '')
sql-run $sql | ignore
{
job_id: $job_id
requested_at: $requested_at
manifest_path: $manifest_path
result_path: $result_path
transcript_path: $transcript_path
session_dir: $session_dir
}
}

View File

@@ -0,0 +1,433 @@
export def home-dir [] {
$nu.home-dir
}
export def data-root [] {
if ('NOTABILITY_DATA_ROOT' in ($env | columns)) {
$env.NOTABILITY_DATA_ROOT
} else {
[$nu.home-dir ".local" "share" "notability-ingest"] | path join
}
}
export def state-root [] {
if ('NOTABILITY_STATE_ROOT' in ($env | columns)) {
$env.NOTABILITY_STATE_ROOT
} else {
[$nu.home-dir ".local" "state" "notability-ingest"] | path join
}
}
export def notes-root [] {
if ('NOTABILITY_NOTES_DIR' in ($env | columns)) {
$env.NOTABILITY_NOTES_DIR
} else {
[$nu.home-dir "Notes"] | path join
}
}
export def webdav-root [] {
if ('NOTABILITY_WEBDAV_ROOT' in ($env | columns)) {
$env.NOTABILITY_WEBDAV_ROOT
} else {
[(data-root) "webdav-root"] | path join
}
}
export def archive-root [] {
if ('NOTABILITY_ARCHIVE_ROOT' in ($env | columns)) {
$env.NOTABILITY_ARCHIVE_ROOT
} else {
[(data-root) "archive"] | path join
}
}
export def render-root [] {
if ('NOTABILITY_RENDER_ROOT' in ($env | columns)) {
$env.NOTABILITY_RENDER_ROOT
} else {
[(data-root) "rendered-pages"] | path join
}
}
export def transcript-root [] {
if ('NOTABILITY_TRANSCRIPT_ROOT' in ($env | columns)) {
$env.NOTABILITY_TRANSCRIPT_ROOT
} else {
[(state-root) "transcripts"] | path join
}
}
export def jobs-root [] {
if ('NOTABILITY_JOBS_ROOT' in ($env | columns)) {
$env.NOTABILITY_JOBS_ROOT
} else {
[(state-root) "jobs"] | path join
}
}
export def queued-root [] {
[(jobs-root) "queued"] | path join
}
export def running-root [] {
[(jobs-root) "running"] | path join
}
export def failed-root [] {
[(jobs-root) "failed"] | path join
}
export def done-root [] {
[(jobs-root) "done"] | path join
}
export def results-root [] {
[(jobs-root) "results"] | path join
}
export def sessions-root [] {
if ('NOTABILITY_SESSIONS_ROOT' in ($env | columns)) {
$env.NOTABILITY_SESSIONS_ROOT
} else {
[(state-root) "sessions"] | path join
}
}
export def qmd-dirty-file [] {
[(state-root) "qmd-dirty"] | path join
}
export def db-path [] {
if ('NOTABILITY_DB_PATH' in ($env | columns)) {
$env.NOTABILITY_DB_PATH
} else {
[(state-root) "db.sqlite"] | path join
}
}
export def now-iso [] {
date now | format date "%Y-%m-%dT%H:%M:%SZ"
}
export def sql-quote [value?: any] {
if $value == null {
"NULL"
} else {
let text = ($value | into string | str replace -a "'" "''")
["'" $text "'"] | str join ''
}
}
export def sql-run [sql: string] {
let database = (db-path)
let result = (^sqlite3 -cmd '.timeout 5000' $database $sql | complete)
if $result.exit_code != 0 {
error make {
msg: $"sqlite3 failed: ($result.stderr | str trim)"
}
}
$result.stdout
}
export def sql-json [sql: string] {
let database = (db-path)
let result = (^sqlite3 -cmd '.timeout 5000' -json $database $sql | complete)
if $result.exit_code != 0 {
error make {
msg: $"sqlite3 failed: ($result.stderr | str trim)"
}
}
let text = ($result.stdout | str trim)
if $text == "" {
[]
} else {
$text | from json
}
}
export def ensure-layout [] {
mkdir (data-root)
mkdir (state-root)
mkdir (notes-root)
mkdir (webdav-root)
mkdir (archive-root)
mkdir (render-root)
mkdir (transcript-root)
mkdir (jobs-root)
mkdir (queued-root)
mkdir (running-root)
mkdir (failed-root)
mkdir (done-root)
mkdir (results-root)
mkdir (sessions-root)
sql-run '
create table if not exists notes (
note_id text primary key,
source_relpath text not null unique,
title text not null,
output_path text not null,
status text not null,
first_seen_at text not null,
last_seen_at text not null,
last_processed_at text,
missing_since text,
deleted_at text,
current_source_hash text,
current_source_size integer,
current_source_mtime text,
current_archive_path text,
latest_version_id text,
last_generated_source_hash text,
last_generated_output_hash text,
conflict_path text,
last_error text
);
create table if not exists versions (
version_id text primary key,
note_id text not null,
seen_at text not null,
archive_path text not null unique,
source_hash text not null,
source_size integer not null,
source_mtime text not null,
source_relpath text not null,
ingest_result text,
session_path text,
foreign key (note_id) references notes (note_id)
);
create table if not exists jobs (
job_id text primary key,
note_id text not null,
operation text not null,
status text not null,
requested_at text not null,
started_at text,
finished_at text,
source_hash text,
job_manifest_path text not null,
result_path text not null,
error_summary text,
foreign key (note_id) references notes (note_id)
);
create table if not exists events (
id integer primary key autoincrement,
note_id text not null,
ts text not null,
kind text not null,
details text,
foreign key (note_id) references notes (note_id)
);
create index if not exists idx_jobs_status_requested_at on jobs(status, requested_at);
create index if not exists idx_versions_note_id_seen_at on versions(note_id, seen_at);
create index if not exists idx_events_note_id_ts on events(note_id, ts);
'
| ignore
}
export def log-event [note_id: string, kind: string, details?: any] {
let payload = if $details == null { null } else { $details | to json }
let note_id_q = (sql-quote $note_id)
let now_q = (sql-quote (now-iso))
let kind_q = (sql-quote $kind)
let payload_q = (sql-quote $payload)
let sql = ([
"insert into events (note_id, ts, kind, details) values ("
$note_id_q
", "
$now_q
", "
$kind_q
", "
$payload_q
");"
] | str join '')
sql-run $sql | ignore
}
export def slugify [value: string] {
let slug = (
$value
| str downcase
| str replace -r '[^a-z0-9]+' '-'
| str replace -r '^-+' ''
| str replace -r '-+$' ''
)
if $slug == '' {
'note'
} else {
$slug
}
}
export def sha256 [file: path] {
(^sha256sum $file | lines | first | split row ' ' | first)
}
export def parse-output-frontmatter [file: path] {
if not ($file | path exists) {
{}
} else {
let content = (open --raw $file)
if not ($content | str starts-with "---\n") {
{}
} else {
let rest = ($content | str substring 4..)
let end = ($rest | str index-of "\n---\n")
if $end == null {
{}
} else {
let block = ($rest | str substring 0..($end - 1))
$block
| lines
| where ($it | str contains ':')
| reduce --fold {} {|line, acc|
let idx = ($line | str index-of ':')
if $idx == null {
$acc
} else {
let key = ($line | str substring 0..($idx - 1) | str trim)
let value = ($line | str substring ($idx + 1).. | str trim)
$acc | upsert $key $value
}
}
}
}
}
}
export def zk-generated-note-path [title: string] {
let root = (notes-root)
let effective_title = if ($title | str trim) == '' {
'Imported note'
} else {
$title
}
let result = (
^zk --notebook-dir $root --working-dir $root new $root --no-input --title $effective_title --print-path --dry-run
| complete
)
if $result.exit_code != 0 {
error make {
msg: $"zk failed to generate a note path: ($result.stderr | str trim)"
}
}
let path_text = ($result.stderr | str trim)
if $path_text == '' {
error make {
msg: 'zk did not return a generated note path'
}
}
$path_text
| lines
| last
| str trim
}
export def new-note-id [] {
let suffix = (random uuid | str replace -a '-' '')
$"ntl_($suffix)"
}
export def new-job-id [] {
let suffix = (random uuid | str replace -a '-' '')
$"job_($suffix)"
}
export def new-version-id [] {
let suffix = (random uuid | str replace -a '-' '')
$"ver_($suffix)"
}
export def archive-path-for [note_id: string, source_hash: string, source_relpath: string] {
let stamp = (date now | format date "%Y-%m-%dT%H-%M-%SZ")
let short = ($source_hash | str substring 0..11)
let directory = [(archive-root) $note_id] | path join
let parsed = ($source_relpath | path parse)
let extension = if (($parsed.extension? | default '') | str trim) == '' {
'bin'
} else {
($parsed.extension | str downcase)
}
mkdir $directory
[$directory $"($stamp)-($short).($extension)"] | path join
}
export def transcript-path-for [note_id: string, job_id: string] {
let directory = [(transcript-root) $note_id] | path join
mkdir $directory
[$directory $"($job_id).md"] | path join
}
export def result-path-for [job_id: string] {
[(results-root) $"($job_id).json"] | path join
}
export def manifest-path-for [job_id: string, status: string] {
let root = match $status {
'queued' => (queued-root)
'running' => (running-root)
'failed' => (failed-root)
'done' => (done-root)
_ => (queued-root)
}
[$root $"($job_id).json"] | path join
}
export def note-output-path [title: string] {
zk-generated-note-path $title
}
export def is-supported-source-path [path: string] {
let lower = ($path | str downcase)
(($lower | str ends-with '.pdf') or ($lower | str ends-with '.png'))
}
export def is-ignored-path [relpath: string] {
let lower = ($relpath | str downcase)
let hidden = (($lower | str contains '/.') or ($lower | str starts-with '.'))
let temp = (($lower | str contains '/~') or ($lower | str ends-with '.tmp') or ($lower | str ends-with '.part'))
let conflict = ($lower | str contains '.sync-conflict')
($hidden or $temp or $conflict)
}
export def scan-source-files [] {
let root = (webdav-root)
if not ($root | path exists) {
[]
} else {
let files = ([
(glob $"($root)/**/*.pdf")
(glob $"($root)/**/*.PDF")
(glob $"($root)/**/*.png")
(glob $"($root)/**/*.PNG")
] | flatten)
$files
| sort
| uniq
| each {|file|
let relpath = ($file | path relative-to $root)
if ((is-ignored-path $relpath) or not (is-supported-source-path $file)) {
null
} else {
let stat = (ls -l $file | first)
{
source_path: $file
source_relpath: $relpath
source_size: $stat.size
source_mtime: ($stat.modified | format date "%Y-%m-%dT%H:%M:%SZ")
title: (($relpath | path parse).stem)
}
}
}
| where $it != null
}
}

View File

@@ -0,0 +1,387 @@
#!/usr/bin/env nu
use ./lib.nu *
use ./jobs.nu [archive-and-version, enqueue-job]
const settle_window = 45sec
const delete_grace = 15min
def settle-remaining [source_mtime: string] {
let modified = ($source_mtime | into datetime)
let age = ((date now) - $modified)
if $age >= $settle_window {
0sec
} else {
$settle_window - $age
}
}
def is-settled [source_mtime: string] {
let modified = ($source_mtime | into datetime)
((date now) - $modified) >= $settle_window
}
def log-job-enqueued [note_id: string, job_id: string, operation: string, source_hash: string, archive_path: string] {
log-event $note_id 'job-enqueued' {
job_id: $job_id
operation: $operation
source_hash: $source_hash
archive_path: $archive_path
}
}
def find-rename-candidate [source_hash: string] {
sql-json $"
select *
from notes
where current_source_hash = (sql-quote $source_hash)
and status != 'active'
and status != 'failed'
and status != 'conflict'
order by last_seen_at desc
limit 1;
"
}
def touch-note [note_id: string, source_size: any, source_mtime: string, status: string = 'active'] {
let source_size_int = ($source_size | into int)
let now_q = (sql-quote (now-iso))
let source_mtime_q = (sql-quote $source_mtime)
let status_q = (sql-quote $status)
let note_id_q = (sql-quote $note_id)
sql-run $"
update notes
set last_seen_at = ($now_q),
current_source_size = ($source_size_int),
current_source_mtime = ($source_mtime_q),
status = ($status_q)
where note_id = ($note_id_q);
"
| ignore
}
def process-existing [note: record, source: record] {
let title = $source.title
let note_id = ($note | get note_id)
let note_status = ($note | get status)
let source_size_int = ($source.source_size | into int)
if not (is-settled $source.source_mtime) {
touch-note $note_id $source_size_int $source.source_mtime $note_status
return
}
let previous_size = ($note.current_source_size? | default (-1))
let previous_mtime = ($note.current_source_mtime? | default '')
let size_changed = ($previous_size != $source_size_int)
let mtime_changed = ($previous_mtime != $source.source_mtime)
let needs_ingest = (($note.last_generated_source_hash? | default '') != ($note.current_source_hash? | default ''))
let hash_needed = ($note.current_source_hash? | default null) == null or $size_changed or $mtime_changed or ($note_status != 'active') or $needs_ingest
if not $hash_needed {
let now_q = (sql-quote (now-iso))
let title_q = (sql-quote $title)
let note_id_q = (sql-quote $note_id)
sql-run $"
update notes
set last_seen_at = ($now_q),
status = 'active',
title = ($title_q),
missing_since = null,
deleted_at = null
where note_id = ($note_id_q);
"
| ignore
return
}
let source_hash = (sha256 $source.source_path)
if ($source_hash == ($note.current_source_hash? | default '')) {
let now_q = (sql-quote (now-iso))
let title_q = (sql-quote $title)
let source_mtime_q = (sql-quote $source.source_mtime)
let note_id_q = (sql-quote $note_id)
let next_status = if $note_status == 'failed' { 'failed' } else { 'active' }
sql-run $"
update notes
set last_seen_at = ($now_q),
title = ($title_q),
status = (sql-quote $next_status),
missing_since = null,
deleted_at = null,
current_source_size = ($source_size_int),
current_source_mtime = ($source_mtime_q)
where note_id = ($note_id_q);
"
| ignore
let should_enqueue = ($note_status == 'failed' or (($note.last_generated_source_hash? | default '') != $source_hash))
if not $should_enqueue {
return
}
let archive_path = if (($note.current_archive_path? | default '') | str trim) == '' {
let version = (archive-and-version $note_id $source.source_path $source.source_relpath $source_size_int $source.source_mtime $source_hash)
let archive_path_q = (sql-quote $version.archive_path)
let version_id_q = (sql-quote $version.version_id)
sql-run $"
update notes
set current_archive_path = ($archive_path_q),
latest_version_id = ($version_id_q)
where note_id = ($note_id_q);
"
| ignore
$version.archive_path
} else {
$note.current_archive_path
}
let runtime_note = ($note | upsert source_path $source.source_path | upsert source_relpath $source.source_relpath | upsert output_path $note.output_path | upsert last_generated_output_hash ($note.last_generated_output_hash? | default null))
let retry_job = (enqueue-job $runtime_note 'upsert' $archive_path $archive_path $source_hash $title)
if $retry_job != null {
log-job-enqueued $note_id $retry_job.job_id 'upsert' $source_hash $archive_path
let reason = if $note_status == 'failed' {
'retry-failed-note'
} else {
'missing-generated-output'
}
log-event $note_id 'job-reenqueued' {
job_id: $retry_job.job_id
reason: $reason
source_hash: $source_hash
archive_path: $archive_path
}
}
return
}
let version = (archive-and-version $note_id $source.source_path $source.source_relpath $source_size_int $source.source_mtime $source_hash)
let now_q = (sql-quote (now-iso))
let title_q = (sql-quote $title)
let source_hash_q = (sql-quote $source_hash)
let source_mtime_q = (sql-quote $source.source_mtime)
let archive_path_q = (sql-quote $version.archive_path)
let version_id_q = (sql-quote $version.version_id)
let note_id_q = (sql-quote $note_id)
sql-run $"
update notes
set last_seen_at = ($now_q),
title = ($title_q),
status = 'active',
missing_since = null,
deleted_at = null,
current_source_hash = ($source_hash_q),
current_source_size = ($source_size_int),
current_source_mtime = ($source_mtime_q),
current_archive_path = ($archive_path_q),
latest_version_id = ($version_id_q),
last_error = null
where note_id = ($note_id_q);
"
| ignore
let runtime_note = ($note | upsert source_path $source.source_path | upsert source_relpath $source.source_relpath | upsert output_path $note.output_path | upsert last_generated_output_hash ($note.last_generated_output_hash? | default null))
let job = (enqueue-job $runtime_note 'upsert' $version.archive_path $version.archive_path $source_hash $title)
if $job != null {
log-job-enqueued $note_id $job.job_id 'upsert' $source_hash $version.archive_path
}
log-event $note_id 'source-updated' {
source_relpath: $source.source_relpath
source_hash: $source_hash
archive_path: $version.archive_path
}
}
def process-new [source: record] {
if not (is-settled $source.source_mtime) {
return
}
let source_hash = (sha256 $source.source_path)
let source_size_int = ($source.source_size | into int)
let rename_candidates = (find-rename-candidate $source_hash)
if not ($rename_candidates | is-empty) {
let rename_candidate = ($rename_candidates | first)
let source_relpath_q = (sql-quote $source.source_relpath)
let title_q = (sql-quote $source.title)
let now_q = (sql-quote (now-iso))
let source_mtime_q = (sql-quote $source.source_mtime)
let note_id_q = (sql-quote $rename_candidate.note_id)
sql-run $"
update notes
set source_relpath = ($source_relpath_q),
title = ($title_q),
last_seen_at = ($now_q),
status = 'active',
missing_since = null,
deleted_at = null,
current_source_size = ($source_size_int),
current_source_mtime = ($source_mtime_q)
where note_id = ($note_id_q);
"
| ignore
log-event $rename_candidate.note_id 'source-renamed' {
from: $rename_candidate.source_relpath
to: $source.source_relpath
}
return
}
let note_id = (new-note-id)
let first_seen_at = (now-iso)
let output_path = (note-output-path $source.title)
let version = (archive-and-version $note_id $source.source_path $source.source_relpath $source_size_int $source.source_mtime $source_hash)
let note_id_q = (sql-quote $note_id)
let source_relpath_q = (sql-quote $source.source_relpath)
let title_q = (sql-quote $source.title)
let output_path_q = (sql-quote $output_path)
let first_seen_q = (sql-quote $first_seen_at)
let source_hash_q = (sql-quote $source_hash)
let source_mtime_q = (sql-quote $source.source_mtime)
let archive_path_q = (sql-quote $version.archive_path)
let version_id_q = (sql-quote $version.version_id)
let sql = ([
"insert into notes (note_id, source_relpath, title, output_path, status, first_seen_at, last_seen_at, current_source_hash, current_source_size, current_source_mtime, current_archive_path, latest_version_id) values ("
$note_id_q
", "
$source_relpath_q
", "
$title_q
", "
$output_path_q
", 'active', "
$first_seen_q
", "
$first_seen_q
", "
$source_hash_q
", "
($source_size_int | into string)
", "
$source_mtime_q
", "
$archive_path_q
", "
$version_id_q
");"
] | str join '')
sql-run $sql | ignore
let note = {
note_id: $note_id
source_relpath: $source.source_relpath
source_path: $source.source_path
output_path: $output_path
last_generated_output_hash: null
}
let job = (enqueue-job $note 'upsert' $version.archive_path $version.archive_path $source_hash $source.title)
if $job != null {
log-job-enqueued $note_id $job.job_id 'upsert' $source_hash $version.archive_path
}
log-event $note_id 'source-discovered' {
source_relpath: $source.source_relpath
source_hash: $source_hash
archive_path: $version.archive_path
output_path: $output_path
}
}
def mark-missing [seen_relpaths: list<string>] {
let notes = (sql-json 'select note_id, source_relpath, status, missing_since from notes;')
for note in $notes {
if ($seen_relpaths | any {|rel| $rel == $note.source_relpath }) {
continue
}
if $note.status == 'active' {
let missing_since = (now-iso)
let missing_since_q = (sql-quote $missing_since)
let note_id_q = (sql-quote $note.note_id)
sql-run $"
update notes
set status = 'source_missing',
missing_since = ($missing_since_q)
where note_id = ($note_id_q);
"
| ignore
log-event $note.note_id 'source-missing' {
source_relpath: $note.source_relpath
}
continue
}
if $note.status == 'source_missing' and ($note.missing_since? | default null) != null {
let missing_since = ($note.missing_since | into datetime)
if ((date now) - $missing_since) >= $delete_grace {
let deleted_at = (now-iso)
let deleted_at_q = (sql-quote $deleted_at)
let note_id_q = (sql-quote $note.note_id)
sql-run $"
update notes
set status = 'source_deleted',
deleted_at = ($deleted_at_q)
where note_id = ($note_id_q);
"
| ignore
log-event $note.note_id 'source-deleted' {
source_relpath: $note.source_relpath
}
}
}
}
}
export def reconcile-run [] {
ensure-layout
mut sources = (scan-source-files)
let unsettled = (
$sources
| each {|source|
{
source_path: $source.source_path
remaining: (settle-remaining $source.source_mtime)
}
}
| where remaining > 0sec
)
if not ($unsettled | is-empty) {
let max_remaining = ($unsettled | get remaining | math max)
print $"Waiting ($max_remaining) for recent Notability uploads to settle"
sleep ($max_remaining + 2sec)
$sources = (scan-source-files)
}
for source in $sources {
let existing_rows = (sql-json $"
select *
from notes
where source_relpath = (sql-quote $source.source_relpath)
limit 1;
")
if (($existing_rows | length) == 0) {
process-new $source
} else {
let existing = ($existing_rows | first)
process-existing ($existing | upsert source_path $source.source_path) $source
}
}
mark-missing ($sources | get source_relpath)
}
def main [] {
reconcile-run
}

View File

@@ -0,0 +1,148 @@
#!/usr/bin/env nu
use ./lib.nu *
use ./jobs.nu [archive-and-version, enqueue-job]
use ./worker.nu [worker-run]
def latest-version [note_id: string] {
sql-json $"
select *
from versions
where note_id = (sql-quote $note_id)
order by seen_at desc
limit 1;
"
| first
}
def existing-active-job [note_id: string, source_hash: string] {
sql-json $"
select job_id
from jobs
where note_id = (sql-quote $note_id)
and source_hash = (sql-quote $source_hash)
and status != 'done'
and status != 'failed'
order by requested_at desc
limit 1;
"
| first
}
def archive-current-source [note: record] {
if not ($note.source_path | path exists) {
error make {
msg: $"Current source path is missing: ($note.source_path)"
}
}
let source_hash = (sha256 $note.source_path)
let source_size = (((ls -l $note.source_path | first).size) | into int)
let source_mtime = (((ls -l $note.source_path | first).modified) | format date "%Y-%m-%dT%H:%M:%SZ")
let version = (archive-and-version $note.note_id $note.source_path $note.source_relpath $source_size $source_mtime $source_hash)
sql-run $"
update notes
set current_source_hash = (sql-quote $source_hash),
current_source_size = ($source_size),
current_source_mtime = (sql-quote $source_mtime),
current_archive_path = (sql-quote $version.archive_path),
latest_version_id = (sql-quote $version.version_id),
last_seen_at = (sql-quote (now-iso)),
status = 'active',
missing_since = null,
deleted_at = null
where note_id = (sql-quote $note.note_id);
"
| ignore
{
input_path: $version.archive_path
archive_path: $version.archive_path
source_hash: $source_hash
}
}
def enqueue-reingest-job [note: record, source_hash: string, input_path: string, archive_path: string, force_overwrite_generated: bool] {
let job = (enqueue-job $note 'reingest' $input_path $archive_path $source_hash $note.title $force_overwrite_generated)
if $job == null {
let existing = (existing-active-job $note.note_id $source_hash)
print $"Already queued: ($existing.job_id? | default 'unknown')"
return
}
log-event $note.note_id 'reingest-enqueued' {
job_id: $job.job_id
source_hash: $source_hash
archive_path: $archive_path
force_overwrite_generated: $force_overwrite_generated
}
print $"Enqueued ($job.job_id) for ($note.note_id)"
try {
worker-run --drain
} catch {|error|
error make {
msg: (($error.msg? | default ($error | to nuon)) | into string)
}
}
}
def main [note_id: string, --latest-source, --latest-archive, --force-overwrite-generated] {
ensure-layout
let note_row = (sql-json $"
select *
from notes
where note_id = (sql-quote $note_id)
limit 1;
" | first)
let note = if $note_row == null {
null
} else {
$note_row | upsert source_path ([ (webdav-root) $note_row.source_relpath ] | path join)
}
if $note == null {
error make {
msg: $"Unknown note id: ($note_id)"
}
}
if $latest_source and $latest_archive {
error make {
msg: 'Choose only one of --latest-source or --latest-archive'
}
}
let source_mode = if $latest_source {
'source'
} else if $latest_archive {
'archive'
} else if ($note.status == 'active' and ($note.source_path | path exists)) {
'source'
} else {
'archive'
}
if $source_mode == 'source' {
let archived = (archive-current-source $note)
enqueue-reingest-job $note $archived.source_hash $archived.input_path $archived.archive_path $force_overwrite_generated
return
}
let version = (latest-version $note.note_id)
if $version == null {
error make {
msg: $"No archived version found for ($note.note_id)"
}
}
enqueue-reingest-job $note $version.source_hash $version.archive_path $version.archive_path $force_overwrite_generated
}

View File

@@ -0,0 +1,202 @@
#!/usr/bin/env nu
use ./lib.nu *
def format-summary [] {
let counts = (sql-json '
select status, count(*) as count
from notes
group by status
order by status;
')
let queue = (sql-json "
select status, count(*) as count
from jobs
where status in ('queued', 'running', 'failed')
group by status
order by status;
")
let lines = [
$"notes db: (db-path)"
$"webdav root: (webdav-root)"
$"notes root: (notes-root)"
''
'notes:'
]
let note_statuses = ('active,source_missing,source_deleted,conflict,failed' | split row ',')
let note_lines = (
$note_statuses
| each {|status|
let row = ($counts | where {|row| ($row | get 'status') == $status } | first)
let count = ($row.count? | default 0)
$" ($status): ($count)"
}
)
let job_statuses = ('queued,running,failed' | split row ',')
let job_lines = (
$job_statuses
| each {|status|
let row = ($queue | where {|row| ($row | get 'status') == $status } | first)
let count = ($row.count? | default 0)
$" ($status): ($count)"
}
)
($lines ++ $note_lines ++ ['' 'jobs:'] ++ $job_lines ++ ['']) | str join "\n"
}
def format-note [note_id: string] {
let note = (sql-json $"
select *
from notes
where note_id = (sql-quote $note_id)
limit 1;
" | first)
if $note == null {
error make {
msg: $"Unknown note id: ($note_id)"
}
}
let jobs = (sql-json $"
select job_id, operation, status, requested_at, started_at, finished_at, source_hash, error_summary
from jobs
where note_id = (sql-quote $note_id)
order by requested_at desc
limit 5;
")
let events = (sql-json $"
select ts, kind, details
from events
where note_id = (sql-quote $note_id)
order by ts desc
limit 10;
")
let output_exists = ($note.output_path | path exists)
let frontmatter = (parse-output-frontmatter $note.output_path)
let lines = [
$"note_id: ($note.note_id)"
$"title: ($note.title)"
$"status: ($note.status)"
$"source_relpath: ($note.source_relpath)"
$"output_path: ($note.output_path)"
$"output_exists: ($output_exists)"
$"managed_by: ($frontmatter.managed_by? | default '')"
$"frontmatter_note_id: ($frontmatter.note_id? | default '')"
$"current_source_hash: ($note.current_source_hash? | default '')"
$"last_generated_output_hash: ($note.last_generated_output_hash? | default '')"
$"current_archive_path: ($note.current_archive_path? | default '')"
$"last_processed_at: ($note.last_processed_at? | default '')"
$"missing_since: ($note.missing_since? | default '')"
$"deleted_at: ($note.deleted_at? | default '')"
$"conflict_path: ($note.conflict_path? | default '')"
$"last_error: ($note.last_error? | default '')"
''
'recent jobs:'
]
let job_lines = if ($jobs | is-empty) {
[' (none)']
} else {
$jobs | each {|job|
$" ($job.job_id) [($job.status)] ($job.operation) requested=($job.requested_at) error=($job.error_summary? | default '')"
}
}
let event_lines = if ($events | is-empty) {
[' (none)']
} else {
$events | each {|event|
$" ($event.ts) ($event.kind) ($event.details? | default '')"
}
}
($lines ++ $job_lines ++ ['' 'recent events:'] ++ $event_lines ++ ['']) | str join "\n"
}
def format-filtered [status: string, label: string] {
let notes = (sql-json $"
select note_id, title, source_relpath, output_path, status, last_error, conflict_path
from notes
where status = (sql-quote $status)
order by last_seen_at desc;
")
let header = [$label]
let body = if ($notes | is-empty) {
[' (none)']
} else {
$notes | each {|note|
let extra = if $status == 'conflict' {
$" conflict_path=($note.conflict_path? | default '')"
} else if $status == 'failed' {
$" last_error=($note.last_error? | default '')"
} else {
''
}
$" ($note.note_id) ($note.title) [($note.status)] source=($note.source_relpath) output=($note.output_path)($extra)"
}
}
($header ++ $body ++ ['']) | str join "\n"
}
def format-queue [] {
let jobs = (sql-json "
select job_id, note_id, operation, status, requested_at, started_at, error_summary
from jobs
where status in ('queued', 'running', 'failed')
order by requested_at asc;
")
let lines = if ($jobs | is-empty) {
['queue' ' (empty)' '']
} else {
['queue'] ++ ($jobs | each {|job|
$" ($job.job_id) note=($job.note_id) [($job.status)] ($job.operation) requested=($job.requested_at) error=($job.error_summary? | default '')"
}) ++ ['']
}
$lines | str join "\n"
}
def main [note_id?: string, --failed, --queue, --deleted, --conflicts] {
ensure-layout
if $queue {
print (format-queue)
return
}
if $failed {
print (format-filtered 'failed' 'failed notes')
return
}
if $deleted {
print (format-filtered 'source_deleted' 'deleted notes')
return
}
if $conflicts {
print (format-filtered 'conflict' 'conflict notes')
return
}
if $note_id != null {
print (format-note $note_id)
return
}
print (format-summary)
}

View File

@@ -0,0 +1,58 @@
#!/usr/bin/env nu
use ./lib.nu *
use ./reconcile.nu [reconcile-run]
use ./worker.nu [worker-run]
def error-message [error: any] {
let msg = (($error.msg? | default '') | into string)
if $msg == '' {
$error | to nuon
} else {
$msg
}
}
def run-worker [] {
try {
worker-run --drain
} catch {|error|
print $"worker failed: (error-message $error)"
}
}
def run-sync [] {
run-worker
try {
reconcile-run
} catch {|error|
print $"reconcile failed: (error-message $error)"
return
}
run-worker
}
def main [] {
ensure-layout
let root = (webdav-root)
print $"Watching ($root) for Notability WebDAV updates"
run-sync
^inotifywait -m -r --format '%w%f' -e create -e close_write -e moved_to -e moved_from -e delete -e attrib $root
| lines
| each {|changed_path|
if not (is-supported-source-path $changed_path) {
return
}
print $"Filesystem event for ($changed_path)"
run-sync
}
}

View File

@@ -0,0 +1,36 @@
#!/usr/bin/env nu
use ./lib.nu *
def main [] {
ensure-layout
let root = (webdav-root)
let addr = if ('NOTABILITY_WEBDAV_ADDR' in ($env | columns)) {
$env.NOTABILITY_WEBDAV_ADDR
} else {
'127.0.0.1:9980'
}
let user = if ('NOTABILITY_WEBDAV_USER' in ($env | columns)) {
$env.NOTABILITY_WEBDAV_USER
} else {
'notability'
}
let baseurl = if ('NOTABILITY_WEBDAV_BASEURL' in ($env | columns)) {
$env.NOTABILITY_WEBDAV_BASEURL
} else {
'/'
}
let password_file = if ('NOTABILITY_WEBDAV_PASSWORD_FILE' in ($env | columns)) {
$env.NOTABILITY_WEBDAV_PASSWORD_FILE
} else {
error make {
msg: 'NOTABILITY_WEBDAV_PASSWORD_FILE is required'
}
}
let password = (open --raw $password_file | str trim)
print $"Starting WebDAV on ($addr), serving ($root), base URL ($baseurl)"
run-external rclone 'serve' 'webdav' $root '--addr' $addr '--baseurl' $baseurl '--user' $user '--pass' $password
}

View File

@@ -0,0 +1,506 @@
#!/usr/bin/env nu
use ./lib.nu *
const qmd_debounce = 1min
const idle_sleep = 10sec
const vision_model = 'openai-codex/gpt-5.4'
const transcribe_timeout = '90s'
const normalize_timeout = '60s'
def next-queued-job [] {
sql-json "
select job_id, note_id, operation, job_manifest_path, result_path, source_hash
from jobs
where status = 'queued'
order by requested_at asc
limit 1;
"
| first
}
def maybe-update-qmd [] {
let dirty = (qmd-dirty-file)
if not ($dirty | path exists) {
return
}
let modified = ((ls -l $dirty | first).modified)
if ((date now) - $modified) < $qmd_debounce {
return
}
print 'Running qmd update'
let result = (do {
cd (notes-root)
run-external qmd 'update' | complete
})
if $result.exit_code != 0 {
print $"qmd update failed: ($result.stderr | str trim)"
return
}
rm -f $dirty
}
def write-result [result_path: path, payload: record] {
mkdir ($result_path | path dirname)
($payload | to json --indent 2) | save -f $result_path
}
def error-message [error: any] {
let msg = (($error.msg? | default '') | into string)
if ($msg == '' or $msg == 'External command failed') {
$error | to nuon
} else {
$msg
}
}
def unquote [value?: any] {
if $value == null {
''
} else {
($value | into string | str replace -r '^"(.*)"$' '$1' | str replace -r "^'(.*)'$" '$1')
}
}
def source-format [file: path] {
(([$file] | path parse | first).extension? | default 'bin' | str downcase)
}
def conflict-path-for [output_path: path] {
let parsed = ([$output_path] | path parse | first)
let stamp = ((date now) | format date '%Y-%m-%dT%H-%M-%SZ')
[$parsed.parent $"($parsed.stem).conflict-($stamp).($parsed.extension)"] | path join
}
def find-managed-outputs [note_id: string] {
let root = (notes-root)
if not ($root | path exists) {
[]
} else {
(glob $"($root)/**/*.md")
| where not ($it | str contains '/.')
| where {|file|
let parsed = (parse-output-frontmatter $file)
(unquote ($parsed.managed_by? | default '')) == 'notability-ingest' and (unquote ($parsed.note_id? | default '')) == $note_id
}
| sort
}
}
def resolve-managed-output-path [note_id: string, configured_output_path: path] {
if ($configured_output_path | path exists) {
let parsed = (parse-output-frontmatter $configured_output_path)
let managed_by = (unquote ($parsed.managed_by? | default ''))
let frontmatter_note_id = (unquote ($parsed.note_id? | default ''))
if ($managed_by == 'notability-ingest' and $frontmatter_note_id == $note_id) {
return $configured_output_path
}
}
let discovered = (find-managed-outputs $note_id)
if ($discovered | is-empty) {
$configured_output_path
} else if (($discovered | length) == 1) {
$discovered | first
} else {
error make {
msg: $"Multiple managed note files found for ($note_id): (($discovered | str join ', '))"
}
}
}
def determine-write-target [manifest: record] {
let output_path = (resolve-managed-output-path $manifest.note_id $manifest.output_path)
if not ($output_path | path exists) {
return {
output_path: $output_path
write_path: $output_path
write_mode: 'create'
updated_main_output: true
}
}
let parsed = (parse-output-frontmatter $output_path)
let managed_by = (unquote ($parsed.managed_by? | default ''))
let frontmatter_note_id = (unquote ($parsed.note_id? | default ''))
if ($managed_by == 'notability-ingest' and $frontmatter_note_id == $manifest.note_id) {
return {
output_path: $output_path
write_path: $output_path
write_mode: 'overwrite'
updated_main_output: true
}
}
{
output_path: $output_path
write_path: (conflict-path-for $output_path)
write_mode: 'conflict'
updated_main_output: false
}
}
def build-markdown [manifest: record, normalized: string] {
let body = ($normalized | str trim)
let output_body = if $body == '' {
$"# ($manifest.title)"
} else {
$body
}
let created = ($manifest.requested_at | str substring 0..9)
let updated = ((date now) | format date '%Y-%m-%d')
[
'---'
$"title: ($manifest.title | to json)"
$"created: ($created | to json)"
$"updated: ($updated | to json)"
'source: "notability"'
$"source_transport: (($manifest.source_transport? | default 'webdav') | to json)"
$"source_relpath: ($manifest.source_relpath | to json)"
$"note_id: ($manifest.note_id | to json)"
'managed_by: "notability-ingest"'
$"source_file: ($manifest.archive_path | to json)"
$"source_file_hash: ($'sha256:($manifest.source_hash)' | to json)"
$"source_format: ((source-format $manifest.archive_path) | to json)"
'status: "active"'
'tags:'
' - handwritten'
' - notability'
'---'
''
$output_body
''
] | str join "\n"
}
def render-pages [input_path: path, job_id: string] {
let extension = (([$input_path] | path parse | first).extension? | default '' | str downcase)
if $extension == 'png' {
[ $input_path ]
} else if $extension == 'pdf' {
let render_dir = [(render-root) $job_id] | path join
mkdir $render_dir
let prefix = [$render_dir 'page'] | path join
^pdftoppm -png -r 200 $input_path $prefix
let pages = ((glob $"($render_dir)/*.png") | sort)
if ($pages | is-empty) {
error make {
msg: $"No PNG pages rendered from ($input_path)"
}
}
$pages
} else {
error make {
msg: $"Unsupported Notability input format: ($input_path)"
}
}
}
def call-pi [timeout_window: string, prompt: string, inputs: list<path>, thinking: string] {
let prompt_file = (^mktemp --suffix '.md' | str trim)
$prompt | save -f $prompt_file
let input_refs = ($inputs | each {|input| $'@($input)' })
let prompt_ref = $'@($prompt_file)'
let result = (try {
^timeout $timeout_window pi --model $vision_model --thinking $thinking --no-tools --no-session -p ...$input_refs $prompt_ref | complete
} catch {|error|
rm -f $prompt_file
error make {
msg: (error-message $error)
}
})
rm -f $prompt_file
let output = ($result.stdout | str trim)
if $output != '' {
$output
} else {
let stderr = ($result.stderr | str trim)
if $stderr == '' {
error make {
msg: $"pi returned no output (exit ($result.exit_code))"
}
} else {
error make {
msg: $"pi returned no output (exit ($result.exit_code)): ($stderr)"
}
}
}
}
def ingest-job [manifest: record] {
mkdir $manifest.session_dir
let page_paths = (render-pages $manifest.input_path $manifest.job_id)
let transcribe_prompt = ([
'Transcribe this note into clean Markdown.'
''
'Read it like a human and reconstruct the intended reading order and structure.'
''
'Do not preserve handwritten layout literally.'
''
'Handwritten line breaks, word stacking, font size changes, and spacing are not semantic structure by default.'
''
'If adjacent handwritten lines clearly belong to one sentence or short phrase, merge them into normal prose with spaces instead of separate Markdown lines.'
''
'Only keep separate lines or blank lines when there is clear evidence of separate paragraphs, headings, list items, checkboxes, or other distinct blocks.'
''
'Keep headings, lists, and paragraphs when they are genuinely present.'
''
'Do not summarize. Do not add commentary. Return Markdown only.'
] | str join "\n")
print $"Transcribing ($manifest.job_id) with page count ($page_paths | length)"
let transcript = (call-pi $transcribe_timeout $transcribe_prompt $page_paths 'low')
mkdir ($manifest.transcript_path | path dirname)
$"($transcript)\n" | save -f $manifest.transcript_path
let normalize_prompt = ([
'Rewrite the attached transcription into clean Markdown.'
''
'Preserve the same content and intended structure.'
''
'Collapse layout-only line breaks from handwriting.'
''
'If short adjacent lines are really one sentence or phrase, join them with spaces instead of keeping one line per handwritten row.'
''
'Use separate lines only for real headings, list items, checkboxes, or distinct paragraphs.'
''
'Do not summarize. Return Markdown only.'
] | str join "\n")
print $"Normalizing ($manifest.job_id)"
let normalized = (call-pi $normalize_timeout $normalize_prompt [ $manifest.transcript_path ] 'off')
let markdown = (build-markdown $manifest $normalized)
let target = (determine-write-target $manifest)
mkdir ($target.write_path | path dirname)
$markdown | save -f $target.write_path
{
success: true
job_id: $manifest.job_id
note_id: $manifest.note_id
archive_path: $manifest.archive_path
source_hash: $manifest.source_hash
session_dir: $manifest.session_dir
output_path: $target.output_path
output_hash: (if $target.updated_main_output { sha256 $target.write_path } else { null })
conflict_path: (if $target.write_mode == 'conflict' { $target.write_path } else { null })
write_mode: $target.write_mode
updated_main_output: $target.updated_main_output
transcript_path: $manifest.transcript_path
}
}
def mark-failure [job: record, running_path: string, error_summary: string, result?: any] {
let finished_at = (now-iso)
sql-run $"
update jobs
set status = 'failed',
finished_at = (sql-quote $finished_at),
error_summary = (sql-quote $error_summary),
job_manifest_path = (sql-quote (manifest-path-for $job.job_id 'failed'))
where job_id = (sql-quote $job.job_id);
update notes
set status = 'failed',
last_error = (sql-quote $error_summary)
where note_id = (sql-quote $job.note_id);
"
| ignore
if $result != null and ($result.archive_path? | default null) != null {
sql-run $"
update versions
set ingest_result = 'failed',
session_path = (sql-quote ($result.session_dir? | default ''))
where archive_path = (sql-quote $result.archive_path);
"
| ignore
}
let failed_path = (manifest-path-for $job.job_id 'failed')
if ($running_path | path exists) {
mv -f $running_path $failed_path
}
log-event $job.note_id 'job-failed' {
job_id: $job.job_id
error: $error_summary
}
}
def mark-success [job: record, running_path: string, result: record] {
let finished_at = (now-iso)
let note_status = if ($result.write_mode? | default 'write') == 'conflict' {
'conflict'
} else {
'active'
}
let output_path_q = (sql-quote ($result.output_path? | default null))
let output_hash_update = if ($result.updated_main_output? | default false) {
sql-quote ($result.output_hash? | default null)
} else {
'last_generated_output_hash'
}
let source_hash_update = if ($result.updated_main_output? | default false) {
sql-quote ($result.source_hash? | default null)
} else {
'last_generated_source_hash'
}
sql-run $"
update jobs
set status = 'done',
finished_at = (sql-quote $finished_at),
error_summary = null,
job_manifest_path = (sql-quote (manifest-path-for $job.job_id 'done'))
where job_id = (sql-quote $job.job_id);
update notes
set status = (sql-quote $note_status),
output_path = ($output_path_q),
last_processed_at = (sql-quote $finished_at),
last_generated_output_hash = ($output_hash_update),
last_generated_source_hash = ($source_hash_update),
conflict_path = (sql-quote ($result.conflict_path? | default null)),
last_error = null
where note_id = (sql-quote $job.note_id);
update versions
set ingest_result = 'success',
session_path = (sql-quote ($result.session_dir? | default ''))
where archive_path = (sql-quote $result.archive_path);
"
| ignore
let done_path = (manifest-path-for $job.job_id 'done')
if ($running_path | path exists) {
mv -f $running_path $done_path
}
^touch (qmd-dirty-file)
log-event $job.note_id 'job-finished' {
job_id: $job.job_id
write_mode: ($result.write_mode? | default 'write')
output_path: ($result.output_path? | default '')
conflict_path: ($result.conflict_path? | default '')
}
}
def recover-running-jobs [] {
let jobs = (sql-json "
select job_id, note_id, job_manifest_path, result_path
from jobs
where status = 'running'
order by started_at asc;
")
for job in $jobs {
let running_path = (manifest-path-for $job.job_id 'running')
let result = if ($job.result_path | path exists) {
open $job.result_path
} else {
null
}
mark-failure $job $running_path 'worker interrupted before completion' $result
}
}
def process-job [job: record] {
let running_path = (manifest-path-for $job.job_id 'running')
mv -f $job.job_manifest_path $running_path
sql-run $"
update jobs
set status = 'running',
started_at = (sql-quote (now-iso)),
job_manifest_path = (sql-quote $running_path)
where job_id = (sql-quote $job.job_id);
"
| ignore
print $"Processing ($job.job_id) for ($job.note_id)"
let manifest = (open $running_path)
try {
let result = (ingest-job $manifest)
write-result $job.result_path $result
mark-success $job $running_path $result
} catch {|error|
let message = (error-message $error)
let result = {
success: false
job_id: $manifest.job_id
note_id: $manifest.note_id
archive_path: $manifest.archive_path
source_hash: $manifest.source_hash
session_dir: $manifest.session_dir
error: $message
}
write-result $job.result_path $result
mark-failure $job $running_path $message $result
}
}
def drain-queued-jobs [] {
loop {
let job = (next-queued-job)
if $job == null {
maybe-update-qmd
break
}
process-job $job
maybe-update-qmd
}
}
export def worker-run [--drain] {
ensure-layout
recover-running-jobs
if $drain {
drain-queued-jobs
return
}
while true {
let job = (next-queued-job)
if $job == null {
maybe-update-qmd
sleep $idle_sleep
continue
}
process-job $job
maybe-update-qmd
}
}
def main [--drain] {
worker-run --drain=$drain
}

View File

@@ -31,6 +31,7 @@
./_parts/tahani/adguardhome.nix
./_parts/tahani/cache.nix
./_parts/tahani/networking.nix
./_parts/tahani/notability.nix
./_parts/tahani/paperless.nix
];

View File

@@ -26,6 +26,8 @@
(import ./_overlays/pi-harness.nix {inherit inputs;})
# pi-mcp-adapter
(import ./_overlays/pi-mcp-adapter.nix {inherit inputs;})
# qmd
(import ./_overlays/qmd.nix {inherit inputs;})
# jj-starship (passes through upstream overlay)
(import ./_overlays/jj-starship.nix {inherit inputs;})
# zjstatus

View File

@@ -66,18 +66,20 @@
'';
xdg.configFile = {
"glow/glow.yml".text = ''
# style name or JSON path (default "auto")
style: "${config.xdg.configHome}/glow/rose-pine-dawn.json"
# mouse support (TUI-mode only)
mouse: false
# use pager to display markdown
pager: false
# word-wrap at width
width: 80
# show all files, including hidden and ignored.
all: false
'';
"glow/glow.yml".text =
lib.concatStringsSep "\n" [
"# style name or JSON path (default \"auto\")"
"style: \"${config.xdg.configHome}/glow/rose-pine-dawn.json\""
"# mouse support (TUI-mode only)"
"mouse: false"
"# use pager to display markdown"
"pager: false"
"# word-wrap at width"
"width: 80"
"# show all files, including hidden and ignored."
"all: false"
""
];
"glow/rose-pine-dawn.json".source = ./_terminal/rose-pine-dawn-glow.json;
};

View File

@@ -0,0 +1,30 @@
{
"data": "ENC[AES256_GCM,data:qZCh11bq1W7FwXMrDX5KMOQFsgsKgbhimZ4TDNvv1BDU,iv:PJJJB5uyhuTUSA4doQ6h6qMbmPgerPv+FfsJ0f20kYY=,tag:lXpit9T7K2rGUu1zsJH6dg==,type:str]",
"sops": {
"age": [
{
"recipient": "age1xate984yhl9qk9d4q99pyxmzz48sq56nfhu8weyzkgum4ed5tc5shjmrs7",
"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBvNXBjN3RZcGd2R2MyQWhR\nenBpa3VHMFhkWDEveUtmZWtPSk01QkhUVFJFCnRSc3ZGdFFjVDJnbHpwKzZ1TUdI\nZUdWQzM2bmZ1RUl4UVpCbDJoL0RkQncKLS0tIHRxUzFiaS8wekdCQ0Z0dTMxSnZ0\nS0UycFNMSUJHcVlkR2JZNlZsbldoaUkKe4EaYIquhABMEywizJXzEVEM1JbEwFqU\nAmQ6R+p4mNgaR5HCrnINQId3qqVfsP2UDqPDepERZIA0V2E5h9ckfQ==\n-----END AGE ENCRYPTED FILE-----\n"
},
{
"recipient": "age1njjegjjdqzfnrr54f536yl4lduqgna3wuv7ef6vtl9jw5cju0grsgy62tm",
"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBBZ1hYenpVTm1lTFdjTEJj\nTUN5MzNtbzdWNzQ2VE9tRlJJRVRYTUtLOXpnCnlLWTZPNGE5NDlwRHhWSnlTNUhv\nc3VZVklEZDB5dXlFc01wcEQxckl0NjgKLS0tIEE5T2JmNlJaYkZpWkhYdDhPSTlW\nei96YmhUWUZ2enVnRjhKOVlNZmNHa3cKxaHBtCwLDLNcscptlDk6ta/i491lLPt6\nOh/RtbkxtJ02cahIsKgajspOElx8u2Nb3/lmK51JbUIexH9TDQ+3tg==\n-----END AGE ENCRYPTED FILE-----\n"
},
{
"recipient": "age187jl7e4k9n4guygkmpuqzeh0wenefwrfkpvuyhvwjrjwxqpzassqq3x67j",
"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBJbFFpQzB2OU9jYUZlL2Nl\nOEZ0WGcyR1BpSmZGU0Vxa0N6WGpCbXBXZGxJCnlLK0JJWElndC9KRGN5d1NNd0tj\nUkExQ0tTSGRKQjJHUGtaWUtKS285MU0KLS0tIGI5cWtVcW43b2Q5VXRidllzamtB\nV1IxYnN1KzdaaXdvWG96a2VkZ0ZvWGsKxdbXwbgFIc3/3VjwUJ1A+cX0oaT+oojz\nrI9Dmk782U/dQrcMv1lRBIWWtAdAqS6GiQ1aUKk5aHpuHOZeHHFjMw==\n-----END AGE ENCRYPTED FILE-----\n"
},
{
"recipient": "age1ez6j3r5wdp0tjy7n5qzv5vfakdc2nh2zeu388zu7a80l0thv052syxq5e2",
"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA0aTgwQ3ZEVG41eW9MQ1RX\nSElRdkdvL21kZ2ZLeGNPbGJiNll5WjdsM2gwCmJQVmJjWEJBaVhEKzJqYWlib2JX\ndWRzSE9QTVQ1c004dldzR2NtR3pvQlUKLS0tIEsvZDNnNWJJaWZyOCtYUEs1eklh\nNXl2dUM0amVtSmdjTy83ZzBSeGp3Q0UKQ/cUYPACFNcxulzW964ftsHjoCBRGB66\nc1e/ObQNM+b+be5UzJi3/gago9CHRzZ3Rp6zE9i5oQBzgLGWlJuPNQ==\n-----END AGE ENCRYPTED FILE-----\n"
},
{
"recipient": "age1tlymdmaukhwupzrhszspp26lgd8s64rw4vu9lwc7gsgrjm78095s9fe9l3",
"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBLNUk5aHBqdEJoYWdaeVlx\nOUkrSXMvRFRmQ29QRE5hWTlHdlcwOUxFRXdRCnE0L1BQdHZDRWRCQUZ2dHQ2Witi\nQ1g5OFFWM2tPT0xEZUZvdXJNdm9aWTgKLS0tIENvM1h1V042L3JHV1pWeDAxdG84\nUTBTZjdHa1lCNGJSRG1iZmtpc1laZTQK/twptPseDi9DM/7NX2F0JO1BEkqklbh1\nxQ1Qwpy4K/P2pFTOBKqDb62DaIALxiGA1Q55dw+fPRSsnL8VcxG8JA==\n-----END AGE ENCRYPTED FILE-----\n"
}
],
"lastmodified": "2026-03-25T11:23:08Z",
"mac": "ENC[AES256_GCM,data:UM0QWfQueExEHRjqNAEIgwpVBjgpd0a6DXxDeRci08qMzTypTlWIofUGMyM1k+J+mUKr3vWMe3q48OwVtUaXnbWimH+8uFEwb5x0e+ayTg+w/C23d+JJmQIX8g5JXtknUAZFNrh3wdZOadYYRr/vDzCKud4lMrmFBKFXsH1DPEI=,iv:kTx8omo8Gt4mTLAs6MoLxj4GizWpxlSXMCTWNlRR5SY=,tag:PB7nMCVxCLRQdhC/eelK/w==,type:str]",
"version": "3.12.2"
}
}