Files
awo-hamburg-intranet/module/knowledgecenter_update/Script/migrate_old_wiki.php
Moritz Weinmann 9817fde8c4 fix(wiki-migration): PDF-Extraktion über alle filegallery-Sitepart-Typen
get_article_files() filterte nur sitepart_id=5, obwohl PDFs auch über
sitepart_id=6 und 10 verknüpft sind. Direkter JOIN auf filegallery_line
mit IN (5,6,10) findet jetzt alle Anhänge korrekt.

Außerdem: Token-sparsamere generate_summary-Logik (Teaser priorisiert,
Text auf 1500 Z. gecappt, nur 1 PDF á 800 Z., maxOutputTokens=150),
--summaries-only-Flag für Nachläufe, sleep(4) für Free-Tier-Rate-Limit.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-06-25 13:33:56 +02:00

462 lines
19 KiB
PHP
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<?php
/**
* Migration: Alte Wiki-Artikel (main_collection) → Neue Wiki-Struktur (knowledgecenter_posts)
*
* CLI-only. Idempotent (über legacy_collection_id). Pro Artikel transaktional.
*
* Aufruf:
* php migrate_old_wiki.php --dry-run --collection-id=1491
* php migrate_old_wiki.php --apply --collection-id=1491 --api-key=KEY
* php migrate_old_wiki.php --apply --limit=5 --api-key=KEY
* php migrate_old_wiki.php --apply (alle Artikel)
*
* Optionen:
* --dry-run Standard. Nur analysieren, nichts schreiben.
* --apply Wirklich migrieren (DB + Dateien).
* --collection-id=N Nur diesen Artikel migrieren.
* --limit=N Maximal N Artikel migrieren (für Tests).
* --api-key=KEY Gemini API Key (oder via GEMINI_API_KEY env).
* --skip-summaries Keine KI-Zusammenfassung generieren.
* --verbose Mehr Details ausgeben.
* --help Diese Hilfe.
*/
if (PHP_SAPI !== 'cli') {
fwrite(STDERR, "Dieses Skript ist CLI-only.\n");
exit(2);
}
$currDir = __DIR__;
$rootDir = dirname(dirname(dirname($currDir)));
$mysydeDir = $rootDir . '/mysyde';
require_once($rootDir . '/vendor/autoload.php');
$envDir = $rootDir . '/config';
$envFilePath = $envDir . '/.env';
if (is_dir($envDir) && file_exists($envFilePath) && is_readable($envFilePath)) {
$dotenv = new \Dotenv\Dotenv($envDir);
$dotenv->load();
}
require_once($mysydeDir . '/common/common_functions.inc.php');
$connEnv = (local_environment()) ? $mysydeDir . '/dc.config.php' : $mysydeDir . '/dc-server.config.php';
require_once($connEnv);
db_connect();
// ------- Args -------
$opts = [
'dry-run' => true,
'apply' => false,
'collection-id' => null,
'limit' => null,
'api-key' => getenv('GEMINI_API_KEY') ?: null,
'skip-summaries' => false,
'summaries-only' => false,
'verbose' => false,
];
foreach (array_slice($argv, 1) as $arg) {
if ($arg === '--help' || $arg === '-h') {
echo "Usage: php migrate_old_wiki.php [--dry-run|--apply] [--collection-id=N] [--limit=N] [--api-key=KEY] [--skip-summaries] [--verbose]\n";
exit(0);
}
if ($arg === '--apply') { $opts['apply'] = true; $opts['dry-run'] = false; continue; }
if ($arg === '--dry-run') { $opts['dry-run'] = true; $opts['apply'] = false; continue; }
if ($arg === '--skip-summaries') { $opts['skip-summaries'] = true; continue; }
if ($arg === '--summaries-only') { $opts['summaries-only'] = true; continue; }
if ($arg === '--verbose') { $opts['verbose'] = true; continue; }
if (preg_match('/^--collection-id=(\d+)$/', $arg, $m)) { $opts['collection-id'] = (int)$m[1]; continue; }
if (preg_match('/^--limit=(\d+)$/', $arg, $m)) { $opts['limit'] = (int)$m[1]; continue; }
if (preg_match('/^--api-key=(.+)$/', $arg, $m)) { $opts['api-key'] = $m[1]; continue; }
fwrite(STDERR, "Unbekannte Option: $arg\n");
exit(2);
}
$dryRun = !$opts['apply'];
$verbose = $opts['verbose'];
$apiKey = $opts['api-key'];
$skipSummary = $opts['skip-summaries'];
$userdataDir = $rootDir . '/userdata';
// ------- Log-Datei -------
$logDir = $userdataDir . '/knowledgecenter_update/migrations';
if (!is_dir($logDir)) @mkdir($logDir, 0775, true);
$logFile = $logDir . '/wiki_migration_' . gmdate('Ymd_His') . '.log';
$logFp = fopen($logFile, 'w');
if (!$logFp) { fwrite(STDERR, "Log-Datei nicht anlegbar: $logFile\n"); exit(3); }
fwrite(STDOUT, "Log: $logFile\n");
// ------- Logging -------
function logmsg(string $msg, string $level = 'INFO'): void {
global $verbose, $logFp;
$line = '[' . gmdate('Y-m-d H:i:s') . " UTC] [$level] $msg\n";
fwrite($logFp, $line);
if ($verbose || in_array($level, ['ERROR', 'FAIL', 'OK', 'WARN', 'HEAD', 'SKIP', 'PROMPT'], true)) {
fwrite(STDOUT, $line);
}
}
// ------- Phase 0: Schema-Vorbereitung -------
logmsg("Prüfe Schema ...", 'HEAD');
mysqli_query($GLOBALS['mysql_con'],
"ALTER TABLE knowledgecenter_posts ADD COLUMN IF NOT EXISTS legacy_collection_id INT NULL");
mysqli_query($GLOBALS['mysql_con'],
"CREATE UNIQUE INDEX IF NOT EXISTS idx_legacy_collection ON knowledgecenter_posts(legacy_collection_id)");
// ------- Hilfsfunktionen -------
/**
* Returns ['teaser' => string, 'content' => string]
* setup_content_id=55 (kc_teaser, data-field) → teaser
* setup_content_id=58 (kc_article, textcontent_header) → content
*/
function assemble_content(int $cid): array {
$sql = "SELECT mcl.main_collection_setup_content_id, mcl.main_sitepart_id,
mcl.main_sitepart_header_id, mcl.data
FROM main_collection_link mcl
WHERE mcl.main_collection_id = $cid
ORDER BY mcl.id";
$res = mysqli_query($GLOBALS['mysql_con'], $sql);
$teaser = '';
$content = '';
while ($row = mysqli_fetch_assoc($res)) {
$scid = (int)$row['main_collection_setup_content_id'];
if ($scid === 55) {
// Teaser: plain text im data-Feld, keine HTML-Tags
$text = trim($row['data'] ?? '');
if ($text !== '') {
$teaser .= ($teaser !== '' ? ' ' : '') . $text;
}
} elseif ($row['main_sitepart_id'] == 1 && $row['main_sitepart_header_id'] > 0) {
// Artikeltext: HTML aus textcontent_header (setup_content_id=58 oder ähnlich)
$hid = (int)$row['main_sitepart_header_id'];
$hres = mysqli_query($GLOBALS['mysql_con'],
"SELECT content FROM textcontent_header WHERE id = $hid LIMIT 1");
if ($hrow = mysqli_fetch_assoc($hres)) {
$content .= $hrow['content'];
}
}
}
return ['teaser' => $teaser, 'content' => $content];
}
function get_article_files(int $cid): array {
// sitepart_id 5, 6, 10 sind filegallery-Typen; sitepart_id=1 ist textcontent_header (nicht joinen!)
$sql = "SELECT DISTINCT fl.filename, fl.extension, fl.description
FROM main_collection_link mcl
JOIN filegallery_line fl ON fl.header_id = mcl.main_sitepart_header_id
WHERE mcl.main_collection_id = $cid
AND mcl.main_sitepart_id IN (5, 6, 10)
AND mcl.main_sitepart_header_id > 0
AND fl.filename != ''
ORDER BY fl.sorting";
$res = mysqli_query($GLOBALS['mysql_con'], $sql);
$files = [];
while ($row = mysqli_fetch_assoc($res)) {
$files[] = $row;
}
return $files;
}
function extract_pdf_text(string $filePath): string {
if (!file_exists($filePath) || filesize($filePath) === 0) return '';
try {
$parser = new \Smalot\PdfParser\Parser();
$pdf = $parser->parseFile($filePath);
$text = trim($pdf->getText());
return preg_replace('/\s+/', ' ', $text);
} catch (\Throwable $e) {
return '';
}
}
function generate_summary(string $htmlContent, array $files, string $userdataDir, string $apiKey, string $title, string $teaser = ''): ?string {
$parts = [];
// Teaser bevorzugen: bereits plain text, sehr kurz → spart Input-Tokens
if (strlen($teaser) > 30) {
$parts[] = $teaser;
}
// Fließtext: HTML entfernen, auf 1500 Zeichen cappen
$plainText = html_entity_decode(strip_tags($htmlContent), ENT_HTML5 | ENT_QUOTES, 'UTF-8');
$plainText = trim(preg_replace('/\s+/', ' ', $plainText));
if (strlen($plainText) > 30) {
$parts[] = mb_substr($plainText, 0, 1500);
}
// PDFs nur wenn sonst kein Inhalt vorhanden, auf 800 Zeichen begrenzen
if (empty($parts)) {
foreach ($files as $file) {
if (strtolower($file['extension']) !== 'pdf') continue;
$pdfText = extract_pdf_text($userdataDir . '/' . $file['filename']);
if (strlen($pdfText) > 50) {
$label = $file['description'] ?: basename($file['filename']);
$parts[] = 'Anhang "' . $label . '": ' . mb_substr($pdfText, 0, 800);
break; // ein PDF reicht
}
}
}
if (empty($parts)) {
return null;
}
$prompt = "Fasse in 2-3 deutschen Sätzen zusammen. Nur Zusammenfassung.\nTitel: $title\n\n"
. implode("\n\n", $parts);
logmsg(" PROMPT (" . strlen($prompt) . " Zeichen)", 'PROMPT');
$payload = json_encode([
'contents' => [['parts' => [['text' => $prompt]]]],
'generationConfig' => [
'maxOutputTokens' => 150,
'temperature' => 0.2,
'thinkingConfig' => ['thinkingBudget' => 0],
],
]);
$url = 'https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash:generateContent?key=' . urlencode($apiKey);
$resp = '';
$httpCode = 0;
$attempts = [[0, null], [5, 503], [15, 503], [30, 429], [60, 429]];
foreach ($attempts as [$wait, $label]) {
if ($wait > 0) {
logmsg(" Gemini $httpCode warte {$wait}s (retry) ...", 'WARN');
sleep($wait);
}
$ch = curl_init($url);
curl_setopt_array($ch, [
CURLOPT_POST => true,
CURLOPT_RETURNTRANSFER => true,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTPHEADER => ['Content-Type: application/json'],
CURLOPT_POSTFIELDS => $payload,
]);
$resp = curl_exec($ch);
$httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE);
curl_close($ch);
if ($httpCode !== 503 && $httpCode !== 429) break;
}
if ($httpCode !== 200) {
logmsg("Gemini API Fehler HTTP $httpCode: " . substr($resp, 0, 200), 'WARN');
return null;
}
$data = json_decode($resp, true);
$summary = trim($data['candidates'][0]['content']['parts'][0]['text'] ?? '');
if (strlen($summary) < 30) {
logmsg(" Gemini Roh-Antwort: " . substr($resp, 0, 500), 'WARN');
}
return $summary;
}
// ------- Permissions aus main_collection auf Post übertragen -------
function copy_permissions_to_post(int $postId, int $collectionId): void {
$maps = [
['src' => 'main_collection_mandant_link', 'src_col' => 'main_mandant_id', 'dst' => 'knowledgecenter_post_mandant', 'dst_col' => 'mandant_id'],
['src' => 'main_collection_department_link', 'src_col' => 'main_department_id', 'dst' => 'knowledgecenter_post_department', 'dst_col' => 'department_id'],
['src' => 'main_collection_role_link', 'src_col' => 'main_role_id', 'dst' => 'knowledgecenter_post_role', 'dst_col' => 'role_id'],
['src' => 'main_collection_einricht_link', 'src_col' => 'main_einricht_id', 'dst' => 'knowledgecenter_post_einricht', 'dst_col' => 'einricht_id'],
['src' => 'main_collection_fachbereich_link', 'src_col' => 'main_fachbereich_id', 'dst' => 'knowledgecenter_post_fachbereich', 'dst_col' => 'fachbereich_id'],
];
foreach ($maps as $m) {
$sql = "INSERT IGNORE INTO {$m['dst']} (post_id, {$m['dst_col']})
SELECT $postId, {$m['src_col']} FROM {$m['src']} WHERE main_collection_id = $collectionId";
mysqli_query($GLOBALS['mysql_con'], $sql);
}
}
// ------- Artikel laden -------
$whereClause = "mc.main_collection_setup_id = 3 AND mc.is_archived = 0 AND mc.active = 1 AND mc.description != ''";
if ($opts['collection-id'] !== null) {
$whereClause .= " AND mc.id = " . (int)$opts['collection-id'];
}
// ------- Summaries-only Modus: fehlende Summaries für bereits migrierte Posts nachfüllen -------
if ($opts['summaries-only']) {
if (!$apiKey) { fwrite(STDERR, "--summaries-only benötigt --api-key\n"); exit(2); }
$sumSql = "SELECT p.id as post_id, p.legacy_collection_id, pv.id as version_id, pv.title,
pv.teaser, pv.content
FROM knowledgecenter_posts p
JOIN knowledgecenter_post_versions pv ON pv.post_id = p.id AND pv.version_number = 1
WHERE (pv.summary IS NULL OR pv.summary = '')
AND p.legacy_collection_id IS NOT NULL";
if ($opts['collection-id'] !== null) {
$sumSql .= " AND p.legacy_collection_id = " . (int)$opts['collection-id'];
}
if ($opts['limit'] !== null) {
$sumSql .= " LIMIT " . (int)$opts['limit'];
}
$sumResult = mysqli_query($GLOBALS['mysql_con'], $sumSql);
$sumTotal = mysqli_num_rows($sumResult);
logmsg("Summaries-only: $sumTotal Posts ohne Summary", 'HEAD');
$sumStats = ['ok' => 0, 'fail' => 0];
while ($sumRow = mysqli_fetch_assoc($sumResult)) {
$postId = (int)$sumRow['post_id'];
$collId = (int)$sumRow['legacy_collection_id'];
$versionId = (int)$sumRow['version_id'];
$title = $sumRow['title'];
$files = get_article_files($collId);
$teaser = $sumRow['teaser'] ?? '';
$summary = generate_summary($sumRow['content'], $files, $userdataDir, $apiKey, $title, $teaser) ?? '';
if ($summary === '') {
logmsg(" [FAIL] Post $postId \"$title\": leere Summary", 'WARN');
$sumStats['fail']++;
} else {
$sumEsc = mysqli_real_escape_string($GLOBALS['mysql_con'], $summary);
mysqli_query($GLOBALS['mysql_con'],
"UPDATE knowledgecenter_post_versions SET summary = '$sumEsc' WHERE id = $versionId");
logmsg("[OK ] Post $postId \"$title\": " . mb_substr($summary, 0, 80), 'OK');
$sumStats['ok']++;
}
sleep(4); // 15 RPM Free-Tier: mind. 4 Sek. zwischen Requests
}
logmsg(sprintf("Summaries fertig. ok=%d fail=%d", $sumStats['ok'], $sumStats['fail']), 'HEAD');
logmsg("Log-Datei: $logFile", 'HEAD');
fclose($logFp);
exit(0);
}
$sql = "SELECT mc.id, mc.description, mc.creation_date, mc.modified_date,
mc.validity_from, mc.validity_to
FROM main_collection mc
WHERE $whereClause
ORDER BY mc.id";
if ($opts['limit'] !== null) {
$sql .= " LIMIT " . (int)$opts['limit'];
}
$result = mysqli_query($GLOBALS['mysql_con'], $sql);
$total = mysqli_num_rows($result);
logmsg("Modus=" . ($dryRun ? 'DRY-RUN' : 'APPLY') . " Artikel=$total Limit=" .
($opts['limit'] ?? 'alle') . " API=" . ($apiKey && !$skipSummary ? 'ja' : 'nein'), 'HEAD');
$stats = ['ok' => 0, 'skipped' => 0, 'fail' => 0];
while ($row = mysqli_fetch_assoc($result)) {
$cid = (int)$row['id'];
$title = $row['description'];
// Idempotenz-Check
$already = mysqli_query($GLOBALS['mysql_con'],
"SELECT id FROM knowledgecenter_posts WHERE legacy_collection_id = $cid LIMIT 1");
if ($already && mysqli_num_rows($already) > 0) {
logmsg("Artikel $cid \"$title\" bereits migriert, überspringe.", 'SKIP');
$stats['skipped']++;
continue;
}
$parts = assemble_content($cid);
$teaserHtml = $parts['teaser'];
$htmlContent = $parts['content'];
$files = get_article_files($cid);
$htmlLen = strlen($htmlContent);
$catRes = mysqli_query($GLOBALS['mysql_con'],
"SELECT knowledgecenter_category_id FROM knowledgecenter_collection_link WHERE main_collection_id = $cid");
$catIds = [];
while ($cr = mysqli_fetch_assoc($catRes)) {
$catIds[] = (int)$cr['knowledgecenter_category_id'];
}
logmsg("Artikel $cid \"$title\" | Teaser=" . strlen($teaserHtml) . "B | HTML=" . $htmlLen . "B | Dateien=" . count($files) .
" | Kategorien=[" . implode(',', $catIds) . "]");
if ($dryRun) {
$stats['ok']++;
continue;
}
// Summary generieren (vor der Transaktion, API-Call außerhalb)
$summary = '';
if (!$skipSummary && $apiKey) {
$fullContent = $teaserHtml . $htmlContent;
$generated = generate_summary($fullContent, $files, $userdataDir, $apiKey, $title);
$summary = $generated ?? '';
logmsg(" Summary: " . (strlen($summary) > 0 ? '"' . mb_substr($summary, 0, 100) . '"' : '(leer)'));
sleep(4); // 15 RPM Free-Tier: mind. 4 Sek. zwischen Requests
}
// HTML-Entities dekodieren für saubere UTF-8 Speicherung
$cleanHtml = html_entity_decode($htmlContent, ENT_HTML5 | ENT_QUOTES, 'UTF-8');
mysqli_query($GLOBALS['mysql_con'], 'START TRANSACTION');
try {
$titleEsc = mysqli_real_escape_string($GLOBALS['mysql_con'], $title);
$summaryEsc = mysqli_real_escape_string($GLOBALS['mysql_con'], $summary);
$teaserEsc = mysqli_real_escape_string($GLOBALS['mysql_con'], $teaserHtml);
$contentEsc = mysqli_real_escape_string($GLOBALS['mysql_con'], $cleanHtml);
$createdAt = date('Y-m-d H:i:s', (int)$row['creation_date']);
$modifiedAt = date('Y-m-d H:i:s', (int)$row['modified_date']);
$validFrom = $row['validity_from'] ? "'" . $row['validity_from'] . "'" : 'NULL';
$validUntil = $row['validity_to'] ? "'" . $row['validity_to'] . "'" : 'NULL';
// knowledgecenter_posts
$sqlPost = "INSERT INTO knowledgecenter_posts (legacy_collection_id, created_at, modified_at)
VALUES ($cid, '$createdAt', '$modifiedAt')";
if (!mysqli_query($GLOBALS['mysql_con'], $sqlPost)) {
throw new RuntimeException("Post-Insert: " . mysqli_error($GLOBALS['mysql_con']));
}
$postId = (int)mysqli_insert_id($GLOBALS['mysql_con']);
// knowledgecenter_post_versions
$sqlVer = "INSERT INTO knowledgecenter_post_versions
(post_id, version_number, language_id, title, teaser, content, summary, state, modified_at, valid_from, valid_until)
VALUES
($postId, 1, 1, '$titleEsc', '$teaserEsc', '$contentEsc', '$summaryEsc', 1, '$modifiedAt', $validFrom, $validUntil)";
if (!mysqli_query($GLOBALS['mysql_con'], $sqlVer)) {
throw new RuntimeException("Version-Insert: " . mysqli_error($GLOBALS['mysql_con']));
}
// Kategorie-Links
foreach ($catIds as $catId) {
mysqli_query($GLOBALS['mysql_con'],
"INSERT IGNORE INTO knowledgecenter_category_post (category_id, post_id) VALUES ($catId, $postId)");
}
// Permissions aus main_collection auf Post übertragen
copy_permissions_to_post($postId, $cid);
// Dateiverknüpfungen: Original-Pfad beibehalten, keine Kopie
foreach ($files as $file) {
$srcPath = $userdataDir . '/' . $file['filename'];
if (!file_exists($srcPath)) {
logmsg(" Quelldatei nicht gefunden: " . $file['filename'], 'WARN');
continue;
}
$relPath = '/userdata/' . $file['filename'];
$descEsc = mysqli_real_escape_string($GLOBALS['mysql_con'], $file['description'] ?: basename($file['filename']));
$pathEsc = mysqli_real_escape_string($GLOBALS['mysql_con'], $relPath);
mysqli_query($GLOBALS['mysql_con'],
"INSERT INTO knowledgecenter_post_files (post_id, description, path)
VALUES ($postId, '$descEsc', '$pathEsc')");
logmsg(" Datei verknüpft: " . $file['filename']);
}
mysqli_query($GLOBALS['mysql_con'], 'COMMIT');
logmsg("[OK ] Artikel $cid Post $postId \"$title\"", 'OK');
$stats['ok']++;
} catch (\Throwable $e) {
mysqli_query($GLOBALS['mysql_con'], 'ROLLBACK');
logmsg("[FAIL] Artikel $cid \"$title\": " . $e->getMessage(), 'FAIL');
$stats['fail']++;
}
}
logmsg(sprintf("Fertig. ok=%d skipped=%d fail=%d modus=%s",
$stats['ok'], $stats['skipped'], $stats['fail'],
$dryRun ? 'DRY-RUN' : 'APPLY'), 'HEAD');
logmsg("Log-Datei: $logFile", 'HEAD');
fclose($logFp);