mow/awo-hamburg-intranet
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(wiki-migration): PDF-Extraktion über alle filegallery-Sitepart-Typen

Browse Source 
get_article_files() filterte nur sitepart_id=5, obwohl PDFs auch über
sitepart_id=6 und 10 verknüpft sind. Direkter JOIN auf filegallery_line
mit IN (5,6,10) findet jetzt alle Anhänge korrekt.

Außerdem: Token-sparsamere generate_summary-Logik (Teaser priorisiert,
Text auf 1500 Z. gecappt, nur 1 PDF á 800 Z., maxOutputTokens=150),
--summaries-only-Flag für Nachläufe, sleep(4) für Free-Tier-Rate-Limit.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Moritz Weinmann
2026-06-25 13:33:56 +02:00
parent 0bf47247ff
commit 9817fde8c4
1 changed files with 41 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

78
module/knowledgecenter_update/Script/migrate_old_wiki.php
View File

@@ -143,22 +143,19 @@ function assemble_content(int $cid): array {
} }
function get_article_files(int $cid): array { function get_article_files(int $cid): array {
// sitepart_id=5 verweist auf filegallery_header // sitepart_id 5, 6, 10 sind filegallery-Typen; sitepart_id=1 ist textcontent_header (nicht joinen!)
$sql = "SELECT mcl.main_sitepart_header_id $sql = "SELECT DISTINCT fl.filename, fl.extension, fl.description
FROM main_collection_link mcl FROM main_collection_link mcl
WHERE mcl.main_collection_id = $cid AND mcl.main_sitepart_id = 5"; JOIN filegallery_line fl ON fl.header_id = mcl.main_sitepart_header_id
WHERE mcl.main_collection_id = $cid
AND mcl.main_sitepart_id IN (5, 6, 10)
AND mcl.main_sitepart_header_id > 0
AND fl.filename != ''
ORDER BY fl.sorting";
$res = mysqli_query($GLOBALS['mysql_con'], $sql); $res = mysqli_query($GLOBALS['mysql_con'], $sql);
$files = []; $files = [];
while ($row = mysqli_fetch_assoc($res)) { while ($row = mysqli_fetch_assoc($res)) {
$hid = (int)$row['main_sitepart_header_id']; $files[] = $row;
if ($hid === 0) continue;
$fres = mysqli_query($GLOBALS['mysql_con'],
"SELECT filename, extension, description FROM filegallery_line WHERE header_id = $hid ORDER BY sorting");
while ($frow = mysqli_fetch_assoc($fres)) {
if (trim($frow['filename']) !== '') {
$files[] = $frow;
}
}
} }
return $files; return $files;
} }
@@ -175,23 +172,31 @@ function extract_pdf_text(string $filePath): string {
} }
} }
function generate_summary(string $htmlContent, array $files, string $userdataDir, string $apiKey, string $title): ?string { function generate_summary(string $htmlContent, array $files, string $userdataDir, string $apiKey, string $title, string $teaser = ''): ?string {
// Fließtext: HTML vollständig entfernen, keine Zeichenbegrenzung
$plainText = html_entity_decode(strip_tags($htmlContent), ENT_HTML5 | ENT_QUOTES, 'UTF-8');
$plainText = trim(preg_replace('/\s+/', ' ', $plainText));
$parts = []; $parts = [];
if (strlen($plainText) > 30) {
$parts[] = $plainText; // Teaser bevorzugen: bereits plain text, sehr kurz → spart Input-Tokens
if (strlen($teaser) > 30) {
$parts[] = $teaser;
} }
// PDFs: Text extrahieren, auf 1500 Zeichen begrenzen (bei sehr langen PDFs) // Fließtext: HTML entfernen, auf 1500 Zeichen cappen
foreach ($files as $file) { $plainText = html_entity_decode(strip_tags($htmlContent), ENT_HTML5 | ENT_QUOTES, 'UTF-8');
if (strtolower($file['extension']) !== 'pdf') continue; $plainText = trim(preg_replace('/\s+/', ' ', $plainText));
$pdfText = extract_pdf_text($userdataDir . '/' . $file['filename']); if (strlen($plainText) > 30) {
if (strlen($pdfText) > 50) { $parts[] = mb_substr($plainText, 0, 1500);
$label = $file['description'] ?: basename($file['filename']); }
$parts[] = 'Anhang "' . $label . '": ' . mb_substr($pdfText, 0, 1500);
// PDFs nur wenn sonst kein Inhalt vorhanden, auf 800 Zeichen begrenzen
if (empty($parts)) {
foreach ($files as $file) {
if (strtolower($file['extension']) !== 'pdf') continue;
$pdfText = extract_pdf_text($userdataDir . '/' . $file['filename']);
if (strlen($pdfText) > 50) {
$label = $file['description'] ?: basename($file['filename']);
$parts[] = 'Anhang "' . $label . '": ' . mb_substr($pdfText, 0, 800);
break; // ein PDF reicht
}
} }
} }
@@ -199,19 +204,17 @@ function generate_summary(string $htmlContent, array $files, string $userdataDir
return null; return null;
} }
// Kompakter Prompt: Titel + vollständiger Inhalt $prompt = "Fasse in 2-3 deutschen Sätzen zusammen. Nur Zusammenfassung.\nTitel: $title\n\n"
$prompt = "Fasse diesen AWO-Wiki-Beitrag in 2-3 deutschen Sätzen zusammen. Nur Zusammenfassung, kein Intro.\n"
. "Titel: $title\n\n"
. implode("\n\n", $parts); . implode("\n\n", $parts);
logmsg(" PROMPT (" . strlen($prompt) . " Zeichen):\n" . $prompt . "\n--- PROMPT END ---", 'PROMPT'); logmsg(" PROMPT (" . strlen($prompt) . " Zeichen)", 'PROMPT');
$payload = json_encode([ $payload = json_encode([
'contents' => [['parts' => [['text' => $prompt]]]], 'contents' => [['parts' => [['text' => $prompt]]]],
'generationConfig' => [ 'generationConfig' => [
'maxOutputTokens' => 300, 'maxOutputTokens' => 150,
'temperature' => 0.2, 'temperature' => 0.2,
'thinkingConfig' => ['thinkingBudget' => 0], // kein Thinking = alle Token für Antwort 'thinkingConfig' => ['thinkingBudget' => 0],
], ],
]); ]);
@@ -279,7 +282,7 @@ if ($opts['summaries-only']) {
if (!$apiKey) { fwrite(STDERR, "--summaries-only benötigt --api-key\n"); exit(2); } if (!$apiKey) { fwrite(STDERR, "--summaries-only benötigt --api-key\n"); exit(2); }
$sumSql = "SELECT p.id as post_id, p.legacy_collection_id, pv.id as version_id, pv.title, $sumSql = "SELECT p.id as post_id, p.legacy_collection_id, pv.id as version_id, pv.title,
pv.content pv.teaser, pv.content
FROM knowledgecenter_posts p FROM knowledgecenter_posts p
JOIN knowledgecenter_post_versions pv ON pv.post_id = p.id AND pv.version_number = 1 JOIN knowledgecenter_post_versions pv ON pv.post_id = p.id AND pv.version_number = 1
WHERE (pv.summary IS NULL OR pv.summary = '') WHERE (pv.summary IS NULL OR pv.summary = '')
@@ -301,8 +304,9 @@ if ($opts['summaries-only']) {
$versionId = (int)$sumRow['version_id']; $versionId = (int)$sumRow['version_id'];
$title = $sumRow['title']; $title = $sumRow['title'];
$files = get_article_files($collId); $files = get_article_files($collId);
$summary = generate_summary($sumRow['content'], $files, $userdataDir, $apiKey, $title) ?? ''; $teaser = $sumRow['teaser'] ?? '';
$summary = generate_summary($sumRow['content'], $files, $userdataDir, $apiKey, $title, $teaser) ?? '';
if ($summary === '') { if ($summary === '') {
logmsg(" [FAIL] Post $postId \"$title\": leere Summary", 'WARN'); logmsg(" [FAIL] Post $postId \"$title\": leere Summary", 'WARN');
@@ -314,7 +318,7 @@ if ($opts['summaries-only']) {
logmsg("[OK ] Post $postId \"$title\": " . mb_substr($summary, 0, 80), 'OK'); logmsg("[OK ] Post $postId \"$title\": " . mb_substr($summary, 0, 80), 'OK');
$sumStats['ok']++; $sumStats['ok']++;
} }
usleep(200000); sleep(4); // 15 RPM Free-Tier: mind. 4 Sek. zwischen Requests
} }
logmsg(sprintf("Summaries fertig. ok=%d fail=%d", $sumStats['ok'], $sumStats['fail']), 'HEAD'); logmsg(sprintf("Summaries fertig. ok=%d fail=%d", $sumStats['ok'], $sumStats['fail']), 'HEAD');
@@ -380,7 +384,7 @@ while ($row = mysqli_fetch_assoc($result)) {
$generated = generate_summary($fullContent, $files, $userdataDir, $apiKey, $title); $generated = generate_summary($fullContent, $files, $userdataDir, $apiKey, $title);
$summary = $generated ?? ''; $summary = $generated ?? '';
logmsg(" Summary: " . (strlen($summary) > 0 ? '"' . mb_substr($summary, 0, 100) . '"' : '(leer)')); logmsg(" Summary: " . (strlen($summary) > 0 ? '"' . mb_substr($summary, 0, 100) . '"' : '(leer)'));
usleep(200000); // 200ms Pause für API rate limiting sleep(4); // 15 RPM Free-Tier: mind. 4 Sek. zwischen Requests
} }
// HTML-Entities dekodieren für saubere UTF-8 Speicherung // HTML-Entities dekodieren für saubere UTF-8 Speicherung