From 9817fde8c41a6d73cab3ca074b9045620aa3420d Mon Sep 17 00:00:00 2001 From: Moritz Weinmann Date: Thu, 25 Jun 2026 13:33:56 +0200 Subject: [PATCH] =?UTF-8?q?fix(wiki-migration):=20PDF-Extraktion=20=C3=BCb?= =?UTF-8?q?er=20alle=20filegallery-Sitepart-Typen?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit get_article_files() filterte nur sitepart_id=5, obwohl PDFs auch über sitepart_id=6 und 10 verknüpft sind. Direkter JOIN auf filegallery_line mit IN (5,6,10) findet jetzt alle Anhänge korrekt. Außerdem: Token-sparsamere generate_summary-Logik (Teaser priorisiert, Text auf 1500 Z. gecappt, nur 1 PDF á 800 Z., maxOutputTokens=150), --summaries-only-Flag für Nachläufe, sleep(4) für Free-Tier-Rate-Limit. Co-Authored-By: Claude Sonnet 4.6 --- .../Script/migrate_old_wiki.php | 78 ++++++++++--------- 1 file changed, 41 insertions(+), 37 deletions(-) diff --git a/module/knowledgecenter_update/Script/migrate_old_wiki.php b/module/knowledgecenter_update/Script/migrate_old_wiki.php index ddf48fa..767f895 100644 --- a/module/knowledgecenter_update/Script/migrate_old_wiki.php +++ b/module/knowledgecenter_update/Script/migrate_old_wiki.php @@ -143,22 +143,19 @@ function assemble_content(int $cid): array { } function get_article_files(int $cid): array { - // sitepart_id=5 verweist auf filegallery_header - $sql = "SELECT mcl.main_sitepart_header_id + // sitepart_id 5, 6, 10 sind filegallery-Typen; sitepart_id=1 ist textcontent_header (nicht joinen!) + $sql = "SELECT DISTINCT fl.filename, fl.extension, fl.description FROM main_collection_link mcl - WHERE mcl.main_collection_id = $cid AND mcl.main_sitepart_id = 5"; + JOIN filegallery_line fl ON fl.header_id = mcl.main_sitepart_header_id + WHERE mcl.main_collection_id = $cid + AND mcl.main_sitepart_id IN (5, 6, 10) + AND mcl.main_sitepart_header_id > 0 + AND fl.filename != '' + ORDER BY fl.sorting"; $res = mysqli_query($GLOBALS['mysql_con'], $sql); $files = []; while ($row = mysqli_fetch_assoc($res)) { - $hid = (int)$row['main_sitepart_header_id']; - if ($hid === 0) continue; - $fres = mysqli_query($GLOBALS['mysql_con'], - "SELECT filename, extension, description FROM filegallery_line WHERE header_id = $hid ORDER BY sorting"); - while ($frow = mysqli_fetch_assoc($fres)) { - if (trim($frow['filename']) !== '') { - $files[] = $frow; - } - } + $files[] = $row; } return $files; } @@ -175,23 +172,31 @@ function extract_pdf_text(string $filePath): string { } } -function generate_summary(string $htmlContent, array $files, string $userdataDir, string $apiKey, string $title): ?string { - // Fließtext: HTML vollständig entfernen, keine Zeichenbegrenzung - $plainText = html_entity_decode(strip_tags($htmlContent), ENT_HTML5 | ENT_QUOTES, 'UTF-8'); - $plainText = trim(preg_replace('/\s+/', ' ', $plainText)); - +function generate_summary(string $htmlContent, array $files, string $userdataDir, string $apiKey, string $title, string $teaser = ''): ?string { $parts = []; - if (strlen($plainText) > 30) { - $parts[] = $plainText; + + // Teaser bevorzugen: bereits plain text, sehr kurz → spart Input-Tokens + if (strlen($teaser) > 30) { + $parts[] = $teaser; } - // PDFs: Text extrahieren, auf 1500 Zeichen begrenzen (bei sehr langen PDFs) - foreach ($files as $file) { - if (strtolower($file['extension']) !== 'pdf') continue; - $pdfText = extract_pdf_text($userdataDir . '/' . $file['filename']); - if (strlen($pdfText) > 50) { - $label = $file['description'] ?: basename($file['filename']); - $parts[] = 'Anhang "' . $label . '": ' . mb_substr($pdfText, 0, 1500); + // Fließtext: HTML entfernen, auf 1500 Zeichen cappen + $plainText = html_entity_decode(strip_tags($htmlContent), ENT_HTML5 | ENT_QUOTES, 'UTF-8'); + $plainText = trim(preg_replace('/\s+/', ' ', $plainText)); + if (strlen($plainText) > 30) { + $parts[] = mb_substr($plainText, 0, 1500); + } + + // PDFs nur wenn sonst kein Inhalt vorhanden, auf 800 Zeichen begrenzen + if (empty($parts)) { + foreach ($files as $file) { + if (strtolower($file['extension']) !== 'pdf') continue; + $pdfText = extract_pdf_text($userdataDir . '/' . $file['filename']); + if (strlen($pdfText) > 50) { + $label = $file['description'] ?: basename($file['filename']); + $parts[] = 'Anhang "' . $label . '": ' . mb_substr($pdfText, 0, 800); + break; // ein PDF reicht + } } } @@ -199,19 +204,17 @@ function generate_summary(string $htmlContent, array $files, string $userdataDir return null; } - // Kompakter Prompt: Titel + vollständiger Inhalt - $prompt = "Fasse diesen AWO-Wiki-Beitrag in 2-3 deutschen Sätzen zusammen. Nur Zusammenfassung, kein Intro.\n" - . "Titel: $title\n\n" + $prompt = "Fasse in 2-3 deutschen Sätzen zusammen. Nur Zusammenfassung.\nTitel: $title\n\n" . implode("\n\n", $parts); - logmsg(" PROMPT (" . strlen($prompt) . " Zeichen):\n" . $prompt . "\n--- PROMPT END ---", 'PROMPT'); + logmsg(" PROMPT (" . strlen($prompt) . " Zeichen)", 'PROMPT'); $payload = json_encode([ 'contents' => [['parts' => [['text' => $prompt]]]], 'generationConfig' => [ - 'maxOutputTokens' => 300, + 'maxOutputTokens' => 150, 'temperature' => 0.2, - 'thinkingConfig' => ['thinkingBudget' => 0], // kein Thinking = alle Token für Antwort + 'thinkingConfig' => ['thinkingBudget' => 0], ], ]); @@ -279,7 +282,7 @@ if ($opts['summaries-only']) { if (!$apiKey) { fwrite(STDERR, "--summaries-only benötigt --api-key\n"); exit(2); } $sumSql = "SELECT p.id as post_id, p.legacy_collection_id, pv.id as version_id, pv.title, - pv.content + pv.teaser, pv.content FROM knowledgecenter_posts p JOIN knowledgecenter_post_versions pv ON pv.post_id = p.id AND pv.version_number = 1 WHERE (pv.summary IS NULL OR pv.summary = '') @@ -301,8 +304,9 @@ if ($opts['summaries-only']) { $versionId = (int)$sumRow['version_id']; $title = $sumRow['title']; - $files = get_article_files($collId); - $summary = generate_summary($sumRow['content'], $files, $userdataDir, $apiKey, $title) ?? ''; + $files = get_article_files($collId); + $teaser = $sumRow['teaser'] ?? ''; + $summary = generate_summary($sumRow['content'], $files, $userdataDir, $apiKey, $title, $teaser) ?? ''; if ($summary === '') { logmsg(" [FAIL] Post $postId \"$title\": leere Summary", 'WARN'); @@ -314,7 +318,7 @@ if ($opts['summaries-only']) { logmsg("[OK ] Post $postId \"$title\": " . mb_substr($summary, 0, 80), 'OK'); $sumStats['ok']++; } - usleep(200000); + sleep(4); // 15 RPM Free-Tier: mind. 4 Sek. zwischen Requests } logmsg(sprintf("Summaries fertig. ok=%d fail=%d", $sumStats['ok'], $sumStats['fail']), 'HEAD'); @@ -380,7 +384,7 @@ while ($row = mysqli_fetch_assoc($result)) { $generated = generate_summary($fullContent, $files, $userdataDir, $apiKey, $title); $summary = $generated ?? ''; logmsg(" Summary: " . (strlen($summary) > 0 ? '"' . mb_substr($summary, 0, 100) . '"' : '(leer)')); - usleep(200000); // 200ms Pause für API rate limiting + sleep(4); // 15 RPM Free-Tier: mind. 4 Sek. zwischen Requests } // HTML-Entities dekodieren für saubere UTF-8 Speicherung