load(); } require_once($mysydeDir . '/common/common_functions.inc.php'); $connEnv = (local_environment()) ? $mysydeDir . '/dc.config.php' : $mysydeDir . '/dc-server.config.php'; require_once($connEnv); db_connect(); // ------- Args ------- $opts = [ 'dry-run' => true, 'apply' => false, 'collection-id' => null, 'limit' => null, 'api-key' => getenv('GEMINI_API_KEY') ?: null, 'skip-summaries' => false, 'summaries-only' => false, 'verbose' => false, ]; foreach (array_slice($argv, 1) as $arg) { if ($arg === '--help' || $arg === '-h') { echo "Usage: php migrate_old_wiki.php [--dry-run|--apply] [--collection-id=N] [--limit=N] [--api-key=KEY] [--skip-summaries] [--verbose]\n"; exit(0); } if ($arg === '--apply') { $opts['apply'] = true; $opts['dry-run'] = false; continue; } if ($arg === '--dry-run') { $opts['dry-run'] = true; $opts['apply'] = false; continue; } if ($arg === '--skip-summaries') { $opts['skip-summaries'] = true; continue; } if ($arg === '--summaries-only') { $opts['summaries-only'] = true; continue; } if ($arg === '--verbose') { $opts['verbose'] = true; continue; } if (preg_match('/^--collection-id=(\d+)$/', $arg, $m)) { $opts['collection-id'] = (int)$m[1]; continue; } if (preg_match('/^--limit=(\d+)$/', $arg, $m)) { $opts['limit'] = (int)$m[1]; continue; } if (preg_match('/^--api-key=(.+)$/', $arg, $m)) { $opts['api-key'] = $m[1]; continue; } fwrite(STDERR, "Unbekannte Option: $arg\n"); exit(2); } $dryRun = !$opts['apply']; $verbose = $opts['verbose']; $apiKey = $opts['api-key']; $skipSummary = $opts['skip-summaries']; $userdataDir = $rootDir . '/userdata'; // ------- Log-Datei ------- $logDir = $userdataDir . '/knowledgecenter_update/migrations'; if (!is_dir($logDir)) @mkdir($logDir, 0775, true); $logFile = $logDir . '/wiki_migration_' . gmdate('Ymd_His') . '.log'; $logFp = fopen($logFile, 'w'); if (!$logFp) { fwrite(STDERR, "Log-Datei nicht anlegbar: $logFile\n"); exit(3); } fwrite(STDOUT, "Log: $logFile\n"); // ------- Logging ------- function logmsg(string $msg, string $level = 'INFO'): void { global $verbose, $logFp; $line = '[' . gmdate('Y-m-d H:i:s') . " UTC] [$level] $msg\n"; fwrite($logFp, $line); if ($verbose || in_array($level, ['ERROR', 'FAIL', 'OK', 'WARN', 'HEAD', 'SKIP', 'PROMPT'], true)) { fwrite(STDOUT, $line); } } // ------- Phase 0: Schema-Vorbereitung ------- logmsg("Prüfe Schema ...", 'HEAD'); mysqli_query($GLOBALS['mysql_con'], "ALTER TABLE knowledgecenter_posts ADD COLUMN IF NOT EXISTS legacy_collection_id INT NULL"); mysqli_query($GLOBALS['mysql_con'], "CREATE UNIQUE INDEX IF NOT EXISTS idx_legacy_collection ON knowledgecenter_posts(legacy_collection_id)"); // ------- Hilfsfunktionen ------- /** * Returns ['teaser' => string, 'content' => string] * setup_content_id=55 (kc_teaser, data-field) → teaser * setup_content_id=58 (kc_article, textcontent_header) → content */ function assemble_content(int $cid): array { $sql = "SELECT mcl.main_collection_setup_content_id, mcl.main_sitepart_id, mcl.main_sitepart_header_id, mcl.data FROM main_collection_link mcl WHERE mcl.main_collection_id = $cid ORDER BY mcl.id"; $res = mysqli_query($GLOBALS['mysql_con'], $sql); $teaser = ''; $content = ''; while ($row = mysqli_fetch_assoc($res)) { $scid = (int)$row['main_collection_setup_content_id']; if ($scid === 55) { // Teaser: plain text im data-Feld, keine HTML-Tags $text = trim($row['data'] ?? ''); if ($text !== '') { $teaser .= ($teaser !== '' ? ' ' : '') . $text; } } elseif ($row['main_sitepart_id'] == 1 && $row['main_sitepart_header_id'] > 0) { // Artikeltext: HTML aus textcontent_header (setup_content_id=58 oder ähnlich) $hid = (int)$row['main_sitepart_header_id']; $hres = mysqli_query($GLOBALS['mysql_con'], "SELECT content FROM textcontent_header WHERE id = $hid LIMIT 1"); if ($hrow = mysqli_fetch_assoc($hres)) { $content .= $hrow['content']; } } } return ['teaser' => $teaser, 'content' => $content]; } function get_article_files(int $cid): array { // sitepart_id 5, 6, 10 sind filegallery-Typen; sitepart_id=1 ist textcontent_header (nicht joinen!) $sql = "SELECT DISTINCT fl.filename, fl.extension, fl.description FROM main_collection_link mcl JOIN filegallery_line fl ON fl.header_id = mcl.main_sitepart_header_id WHERE mcl.main_collection_id = $cid AND mcl.main_sitepart_id IN (5, 6, 10) AND mcl.main_sitepart_header_id > 0 AND fl.filename != '' ORDER BY fl.sorting"; $res = mysqli_query($GLOBALS['mysql_con'], $sql); $files = []; while ($row = mysqli_fetch_assoc($res)) { $files[] = $row; } return $files; } function extract_pdf_text(string $filePath): string { if (!file_exists($filePath) || filesize($filePath) === 0) return ''; try { $parser = new \Smalot\PdfParser\Parser(); $pdf = $parser->parseFile($filePath); $text = trim($pdf->getText()); return preg_replace('/\s+/', ' ', $text); } catch (\Throwable $e) { return ''; } } function generate_summary(string $htmlContent, array $files, string $userdataDir, string $apiKey, string $title, string $teaser = ''): ?string { $parts = []; // Teaser bevorzugen: bereits plain text, sehr kurz → spart Input-Tokens if (strlen($teaser) > 30) { $parts[] = $teaser; } // Fließtext: HTML entfernen, auf 1500 Zeichen cappen $plainText = html_entity_decode(strip_tags($htmlContent), ENT_HTML5 | ENT_QUOTES, 'UTF-8'); $plainText = trim(preg_replace('/\s+/', ' ', $plainText)); if (strlen($plainText) > 30) { $parts[] = mb_substr($plainText, 0, 1500); } // PDFs nur wenn sonst kein Inhalt vorhanden, auf 800 Zeichen begrenzen if (empty($parts)) { foreach ($files as $file) { if (strtolower($file['extension']) !== 'pdf') continue; $pdfText = extract_pdf_text($userdataDir . '/' . $file['filename']); if (strlen($pdfText) > 50) { $label = $file['description'] ?: basename($file['filename']); $parts[] = 'Anhang "' . $label . '": ' . mb_substr($pdfText, 0, 800); break; // ein PDF reicht } } } if (empty($parts)) { return null; } $prompt = "Fasse in 2-3 deutschen Sätzen zusammen. Nur Zusammenfassung.\nTitel: $title\n\n" . implode("\n\n", $parts); logmsg(" PROMPT (" . strlen($prompt) . " Zeichen)", 'PROMPT'); $payload = json_encode([ 'contents' => [['parts' => [['text' => $prompt]]]], 'generationConfig' => [ 'maxOutputTokens' => 150, 'temperature' => 0.2, 'thinkingConfig' => ['thinkingBudget' => 0], ], ]); $url = 'https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash:generateContent?key=' . urlencode($apiKey); $resp = ''; $httpCode = 0; $attempts = [[0, null], [5, 503], [15, 503], [30, 429], [60, 429]]; foreach ($attempts as [$wait, $label]) { if ($wait > 0) { logmsg(" Gemini $httpCode – warte {$wait}s (retry) ...", 'WARN'); sleep($wait); } $ch = curl_init($url); curl_setopt_array($ch, [ CURLOPT_POST => true, CURLOPT_RETURNTRANSFER => true, CURLOPT_TIMEOUT => 30, CURLOPT_HTTPHEADER => ['Content-Type: application/json'], CURLOPT_POSTFIELDS => $payload, ]); $resp = curl_exec($ch); $httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE); curl_close($ch); if ($httpCode !== 503 && $httpCode !== 429) break; } if ($httpCode !== 200) { logmsg("Gemini API Fehler HTTP $httpCode: " . substr($resp, 0, 200), 'WARN'); return null; } $data = json_decode($resp, true); $summary = trim($data['candidates'][0]['content']['parts'][0]['text'] ?? ''); if (strlen($summary) < 30) { logmsg(" Gemini Roh-Antwort: " . substr($resp, 0, 500), 'WARN'); } return $summary; } // ------- Permissions aus main_collection auf Post übertragen ------- function copy_permissions_to_post(int $postId, int $collectionId): void { $maps = [ ['src' => 'main_collection_mandant_link', 'src_col' => 'main_mandant_id', 'dst' => 'knowledgecenter_post_mandant', 'dst_col' => 'mandant_id'], ['src' => 'main_collection_department_link', 'src_col' => 'main_department_id', 'dst' => 'knowledgecenter_post_department', 'dst_col' => 'department_id'], ['src' => 'main_collection_role_link', 'src_col' => 'main_role_id', 'dst' => 'knowledgecenter_post_role', 'dst_col' => 'role_id'], ['src' => 'main_collection_einricht_link', 'src_col' => 'main_einricht_id', 'dst' => 'knowledgecenter_post_einricht', 'dst_col' => 'einricht_id'], ['src' => 'main_collection_fachbereich_link', 'src_col' => 'main_fachbereich_id', 'dst' => 'knowledgecenter_post_fachbereich', 'dst_col' => 'fachbereich_id'], ]; foreach ($maps as $m) { $sql = "INSERT IGNORE INTO {$m['dst']} (post_id, {$m['dst_col']}) SELECT $postId, {$m['src_col']} FROM {$m['src']} WHERE main_collection_id = $collectionId"; mysqli_query($GLOBALS['mysql_con'], $sql); } } // ------- Artikel laden ------- $whereClause = "mc.main_collection_setup_id = 3 AND mc.is_archived = 0 AND mc.active = 1 AND mc.description != ''"; if ($opts['collection-id'] !== null) { $whereClause .= " AND mc.id = " . (int)$opts['collection-id']; } // ------- Summaries-only Modus: fehlende Summaries für bereits migrierte Posts nachfüllen ------- if ($opts['summaries-only']) { if (!$apiKey) { fwrite(STDERR, "--summaries-only benötigt --api-key\n"); exit(2); } $sumSql = "SELECT p.id as post_id, p.legacy_collection_id, pv.id as version_id, pv.title, pv.teaser, pv.content FROM knowledgecenter_posts p JOIN knowledgecenter_post_versions pv ON pv.post_id = p.id AND pv.version_number = 1 WHERE (pv.summary IS NULL OR pv.summary = '') AND p.legacy_collection_id IS NOT NULL"; if ($opts['collection-id'] !== null) { $sumSql .= " AND p.legacy_collection_id = " . (int)$opts['collection-id']; } if ($opts['limit'] !== null) { $sumSql .= " LIMIT " . (int)$opts['limit']; } $sumResult = mysqli_query($GLOBALS['mysql_con'], $sumSql); $sumTotal = mysqli_num_rows($sumResult); logmsg("Summaries-only: $sumTotal Posts ohne Summary", 'HEAD'); $sumStats = ['ok' => 0, 'fail' => 0]; while ($sumRow = mysqli_fetch_assoc($sumResult)) { $postId = (int)$sumRow['post_id']; $collId = (int)$sumRow['legacy_collection_id']; $versionId = (int)$sumRow['version_id']; $title = $sumRow['title']; $files = get_article_files($collId); $teaser = $sumRow['teaser'] ?? ''; $summary = generate_summary($sumRow['content'], $files, $userdataDir, $apiKey, $title, $teaser) ?? ''; if ($summary === '') { logmsg(" [FAIL] Post $postId \"$title\": leere Summary", 'WARN'); $sumStats['fail']++; } else { $sumEsc = mysqli_real_escape_string($GLOBALS['mysql_con'], $summary); mysqli_query($GLOBALS['mysql_con'], "UPDATE knowledgecenter_post_versions SET summary = '$sumEsc' WHERE id = $versionId"); logmsg("[OK ] Post $postId \"$title\": " . mb_substr($summary, 0, 80), 'OK'); $sumStats['ok']++; } sleep(4); // 15 RPM Free-Tier: mind. 4 Sek. zwischen Requests } logmsg(sprintf("Summaries fertig. ok=%d fail=%d", $sumStats['ok'], $sumStats['fail']), 'HEAD'); logmsg("Log-Datei: $logFile", 'HEAD'); fclose($logFp); exit(0); } $sql = "SELECT mc.id, mc.description, mc.creation_date, mc.modified_date, mc.validity_from, mc.validity_to FROM main_collection mc WHERE $whereClause ORDER BY mc.id"; if ($opts['limit'] !== null) { $sql .= " LIMIT " . (int)$opts['limit']; } $result = mysqli_query($GLOBALS['mysql_con'], $sql); $total = mysqli_num_rows($result); logmsg("Modus=" . ($dryRun ? 'DRY-RUN' : 'APPLY') . " Artikel=$total Limit=" . ($opts['limit'] ?? 'alle') . " API=" . ($apiKey && !$skipSummary ? 'ja' : 'nein'), 'HEAD'); $stats = ['ok' => 0, 'skipped' => 0, 'fail' => 0]; while ($row = mysqli_fetch_assoc($result)) { $cid = (int)$row['id']; $title = $row['description']; // Idempotenz-Check $already = mysqli_query($GLOBALS['mysql_con'], "SELECT id FROM knowledgecenter_posts WHERE legacy_collection_id = $cid LIMIT 1"); if ($already && mysqli_num_rows($already) > 0) { logmsg("Artikel $cid \"$title\" – bereits migriert, überspringe.", 'SKIP'); $stats['skipped']++; continue; } $parts = assemble_content($cid); $teaserHtml = $parts['teaser']; $htmlContent = $parts['content']; $files = get_article_files($cid); $htmlLen = strlen($htmlContent); $catRes = mysqli_query($GLOBALS['mysql_con'], "SELECT knowledgecenter_category_id FROM knowledgecenter_collection_link WHERE main_collection_id = $cid"); $catIds = []; while ($cr = mysqli_fetch_assoc($catRes)) { $catIds[] = (int)$cr['knowledgecenter_category_id']; } logmsg("Artikel $cid \"$title\" | Teaser=" . strlen($teaserHtml) . "B | HTML=" . $htmlLen . "B | Dateien=" . count($files) . " | Kategorien=[" . implode(',', $catIds) . "]"); if ($dryRun) { $stats['ok']++; continue; } // Summary generieren (vor der Transaktion, API-Call außerhalb) $summary = ''; if (!$skipSummary && $apiKey) { $fullContent = $teaserHtml . $htmlContent; $generated = generate_summary($fullContent, $files, $userdataDir, $apiKey, $title); $summary = $generated ?? ''; logmsg(" Summary: " . (strlen($summary) > 0 ? '"' . mb_substr($summary, 0, 100) . '"' : '(leer)')); sleep(4); // 15 RPM Free-Tier: mind. 4 Sek. zwischen Requests } // HTML-Entities dekodieren für saubere UTF-8 Speicherung $cleanHtml = html_entity_decode($htmlContent, ENT_HTML5 | ENT_QUOTES, 'UTF-8'); mysqli_query($GLOBALS['mysql_con'], 'START TRANSACTION'); try { $titleEsc = mysqli_real_escape_string($GLOBALS['mysql_con'], $title); $summaryEsc = mysqli_real_escape_string($GLOBALS['mysql_con'], $summary); $teaserEsc = mysqli_real_escape_string($GLOBALS['mysql_con'], $teaserHtml); $contentEsc = mysqli_real_escape_string($GLOBALS['mysql_con'], $cleanHtml); $createdAt = date('Y-m-d H:i:s', (int)$row['creation_date']); $modifiedAt = date('Y-m-d H:i:s', (int)$row['modified_date']); $validFrom = $row['validity_from'] ? "'" . $row['validity_from'] . "'" : 'NULL'; $validUntil = $row['validity_to'] ? "'" . $row['validity_to'] . "'" : 'NULL'; // knowledgecenter_posts $sqlPost = "INSERT INTO knowledgecenter_posts (legacy_collection_id, created_at, modified_at) VALUES ($cid, '$createdAt', '$modifiedAt')"; if (!mysqli_query($GLOBALS['mysql_con'], $sqlPost)) { throw new RuntimeException("Post-Insert: " . mysqli_error($GLOBALS['mysql_con'])); } $postId = (int)mysqli_insert_id($GLOBALS['mysql_con']); // knowledgecenter_post_versions $sqlVer = "INSERT INTO knowledgecenter_post_versions (post_id, version_number, language_id, title, teaser, content, summary, state, modified_at, valid_from, valid_until) VALUES ($postId, 1, 1, '$titleEsc', '$teaserEsc', '$contentEsc', '$summaryEsc', 1, '$modifiedAt', $validFrom, $validUntil)"; if (!mysqli_query($GLOBALS['mysql_con'], $sqlVer)) { throw new RuntimeException("Version-Insert: " . mysqli_error($GLOBALS['mysql_con'])); } // Kategorie-Links foreach ($catIds as $catId) { mysqli_query($GLOBALS['mysql_con'], "INSERT IGNORE INTO knowledgecenter_category_post (category_id, post_id) VALUES ($catId, $postId)"); } // Permissions aus main_collection auf Post übertragen copy_permissions_to_post($postId, $cid); // Dateiverknüpfungen: Original-Pfad beibehalten, keine Kopie foreach ($files as $file) { $srcPath = $userdataDir . '/' . $file['filename']; if (!file_exists($srcPath)) { logmsg(" Quelldatei nicht gefunden: " . $file['filename'], 'WARN'); continue; } $relPath = '/userdata/' . $file['filename']; $descEsc = mysqli_real_escape_string($GLOBALS['mysql_con'], $file['description'] ?: basename($file['filename'])); $pathEsc = mysqli_real_escape_string($GLOBALS['mysql_con'], $relPath); mysqli_query($GLOBALS['mysql_con'], "INSERT INTO knowledgecenter_post_files (post_id, description, path) VALUES ($postId, '$descEsc', '$pathEsc')"); logmsg(" Datei verknüpft: " . $file['filename']); } mysqli_query($GLOBALS['mysql_con'], 'COMMIT'); logmsg("[OK ] Artikel $cid → Post $postId \"$title\"", 'OK'); $stats['ok']++; } catch (\Throwable $e) { mysqli_query($GLOBALS['mysql_con'], 'ROLLBACK'); logmsg("[FAIL] Artikel $cid \"$title\": " . $e->getMessage(), 'FAIL'); $stats['fail']++; } } logmsg(sprintf("Fertig. ok=%d skipped=%d fail=%d modus=%s", $stats['ok'], $stats['skipped'], $stats['fail'], $dryRun ? 'DRY-RUN' : 'APPLY'), 'HEAD'); logmsg("Log-Datei: $logFile", 'HEAD'); fclose($logFp);