Files
awo-hamburg-intranet/module/knowledgecenter_update/Script/migrate_old_wiki.php

458 lines
19 KiB
PHP
Raw Normal View History

<?php
/**
* Migration: Alte Wiki-Artikel (main_collection) Neue Wiki-Struktur (knowledgecenter_posts)
*
* CLI-only. Idempotent (über legacy_collection_id). Pro Artikel transaktional.
*
* Aufruf:
* php migrate_old_wiki.php --dry-run --collection-id=1491
* php migrate_old_wiki.php --apply --collection-id=1491 --api-key=KEY
* php migrate_old_wiki.php --apply --limit=5 --api-key=KEY
* php migrate_old_wiki.php --apply (alle Artikel)
*
* Optionen:
* --dry-run Standard. Nur analysieren, nichts schreiben.
* --apply Wirklich migrieren (DB + Dateien).
* --collection-id=N Nur diesen Artikel migrieren.
* --limit=N Maximal N Artikel migrieren (für Tests).
* --api-key=KEY Gemini API Key (oder via GEMINI_API_KEY env).
* --skip-summaries Keine KI-Zusammenfassung generieren.
* --verbose Mehr Details ausgeben.
* --help Diese Hilfe.
*/
if (PHP_SAPI !== 'cli') {
fwrite(STDERR, "Dieses Skript ist CLI-only.\n");
exit(2);
}
$currDir = __DIR__;
$rootDir = dirname(dirname(dirname($currDir)));
$mysydeDir = $rootDir . '/mysyde';
require_once($rootDir . '/vendor/autoload.php');
$envDir = $rootDir . '/config';
$envFilePath = $envDir . '/.env';
if (is_dir($envDir) && file_exists($envFilePath) && is_readable($envFilePath)) {
$dotenv = new \Dotenv\Dotenv($envDir);
$dotenv->load();
}
require_once($mysydeDir . '/common/common_functions.inc.php');
$connEnv = (local_environment()) ? $mysydeDir . '/dc.config.php' : $mysydeDir . '/dc-server.config.php';
require_once($connEnv);
db_connect();
// ------- Args -------
$opts = [
'dry-run' => true,
'apply' => false,
'collection-id' => null,
'limit' => null,
'api-key' => getenv('GEMINI_API_KEY') ?: null,
'skip-summaries' => false,
'summaries-only' => false,
'verbose' => false,
];
foreach (array_slice($argv, 1) as $arg) {
if ($arg === '--help' || $arg === '-h') {
echo "Usage: php migrate_old_wiki.php [--dry-run|--apply] [--collection-id=N] [--limit=N] [--api-key=KEY] [--skip-summaries] [--verbose]\n";
exit(0);
}
if ($arg === '--apply') { $opts['apply'] = true; $opts['dry-run'] = false; continue; }
if ($arg === '--dry-run') { $opts['dry-run'] = true; $opts['apply'] = false; continue; }
if ($arg === '--skip-summaries') { $opts['skip-summaries'] = true; continue; }
if ($arg === '--summaries-only') { $opts['summaries-only'] = true; continue; }
if ($arg === '--verbose') { $opts['verbose'] = true; continue; }
if (preg_match('/^--collection-id=(\d+)$/', $arg, $m)) { $opts['collection-id'] = (int)$m[1]; continue; }
if (preg_match('/^--limit=(\d+)$/', $arg, $m)) { $opts['limit'] = (int)$m[1]; continue; }
if (preg_match('/^--api-key=(.+)$/', $arg, $m)) { $opts['api-key'] = $m[1]; continue; }
fwrite(STDERR, "Unbekannte Option: $arg\n");
exit(2);
}
$dryRun = !$opts['apply'];
$verbose = $opts['verbose'];
$apiKey = $opts['api-key'];
$skipSummary = $opts['skip-summaries'];
$userdataDir = $rootDir . '/userdata';
// ------- Log-Datei -------
$logDir = $userdataDir . '/knowledgecenter_update/migrations';
if (!is_dir($logDir)) @mkdir($logDir, 0775, true);
$logFile = $logDir . '/wiki_migration_' . gmdate('Ymd_His') . '.log';
$logFp = fopen($logFile, 'w');
if (!$logFp) { fwrite(STDERR, "Log-Datei nicht anlegbar: $logFile\n"); exit(3); }
fwrite(STDOUT, "Log: $logFile\n");
// ------- Logging -------
function logmsg(string $msg, string $level = 'INFO'): void {
global $verbose, $logFp;
$line = '[' . gmdate('Y-m-d H:i:s') . " UTC] [$level] $msg\n";
fwrite($logFp, $line);
if ($verbose || in_array($level, ['ERROR', 'FAIL', 'OK', 'WARN', 'HEAD', 'SKIP', 'PROMPT'], true)) {
fwrite(STDOUT, $line);
}
}
// ------- Phase 0: Schema-Vorbereitung -------
logmsg("Prüfe Schema ...", 'HEAD');
mysqli_query($GLOBALS['mysql_con'],
"ALTER TABLE knowledgecenter_posts ADD COLUMN IF NOT EXISTS legacy_collection_id INT NULL");
mysqli_query($GLOBALS['mysql_con'],
"CREATE UNIQUE INDEX IF NOT EXISTS idx_legacy_collection ON knowledgecenter_posts(legacy_collection_id)");
// ------- Hilfsfunktionen -------
/**
* Returns ['teaser' => string, 'content' => string]
* setup_content_id=55 (kc_teaser, data-field) teaser
* setup_content_id=58 (kc_article, textcontent_header) content
*/
function assemble_content(int $cid): array {
$sql = "SELECT mcl.main_collection_setup_content_id, mcl.main_sitepart_id,
mcl.main_sitepart_header_id, mcl.data
FROM main_collection_link mcl
WHERE mcl.main_collection_id = $cid
ORDER BY mcl.id";
$res = mysqli_query($GLOBALS['mysql_con'], $sql);
$teaser = '';
$content = '';
while ($row = mysqli_fetch_assoc($res)) {
$scid = (int)$row['main_collection_setup_content_id'];
if ($scid === 55) {
// Teaser: plain text im data-Feld, keine HTML-Tags
$text = trim($row['data'] ?? '');
if ($text !== '') {
$teaser .= ($teaser !== '' ? ' ' : '') . $text;
}
} elseif ($row['main_sitepart_id'] == 1 && $row['main_sitepart_header_id'] > 0) {
// Artikeltext: HTML aus textcontent_header (setup_content_id=58 oder ähnlich)
$hid = (int)$row['main_sitepart_header_id'];
$hres = mysqli_query($GLOBALS['mysql_con'],
"SELECT content FROM textcontent_header WHERE id = $hid LIMIT 1");
if ($hrow = mysqli_fetch_assoc($hres)) {
$content .= $hrow['content'];
}
}
}
return ['teaser' => $teaser, 'content' => $content];
}
function get_article_files(int $cid): array {
// sitepart_id=5 verweist auf filegallery_header
$sql = "SELECT mcl.main_sitepart_header_id
FROM main_collection_link mcl
WHERE mcl.main_collection_id = $cid AND mcl.main_sitepart_id = 5";
$res = mysqli_query($GLOBALS['mysql_con'], $sql);
$files = [];
while ($row = mysqli_fetch_assoc($res)) {
$hid = (int)$row['main_sitepart_header_id'];
if ($hid === 0) continue;
$fres = mysqli_query($GLOBALS['mysql_con'],
"SELECT filename, extension, description FROM filegallery_line WHERE header_id = $hid ORDER BY sorting");
while ($frow = mysqli_fetch_assoc($fres)) {
if (trim($frow['filename']) !== '') {
$files[] = $frow;
}
}
}
return $files;
}
function extract_pdf_text(string $filePath): string {
if (!file_exists($filePath) || filesize($filePath) === 0) return '';
try {
$parser = new \Smalot\PdfParser\Parser();
$pdf = $parser->parseFile($filePath);
$text = trim($pdf->getText());
return preg_replace('/\s+/', ' ', $text);
} catch (\Throwable $e) {
return '';
}
}
function generate_summary(string $htmlContent, array $files, string $userdataDir, string $apiKey, string $title): ?string {
// Fließtext: HTML vollständig entfernen, keine Zeichenbegrenzung
$plainText = html_entity_decode(strip_tags($htmlContent), ENT_HTML5 | ENT_QUOTES, 'UTF-8');
$plainText = trim(preg_replace('/\s+/', ' ', $plainText));
$parts = [];
if (strlen($plainText) > 30) {
$parts[] = $plainText;
}
// PDFs: Text extrahieren, auf 1500 Zeichen begrenzen (bei sehr langen PDFs)
foreach ($files as $file) {
if (strtolower($file['extension']) !== 'pdf') continue;
$pdfText = extract_pdf_text($userdataDir . '/' . $file['filename']);
if (strlen($pdfText) > 50) {
$label = $file['description'] ?: basename($file['filename']);
$parts[] = 'Anhang "' . $label . '": ' . mb_substr($pdfText, 0, 1500);
}
}
if (empty($parts)) {
return null;
}
// Kompakter Prompt: Titel + vollständiger Inhalt
$prompt = "Fasse diesen AWO-Wiki-Beitrag in 2-3 deutschen Sätzen zusammen. Nur Zusammenfassung, kein Intro.\n"
. "Titel: $title\n\n"
. implode("\n\n", $parts);
logmsg(" PROMPT (" . strlen($prompt) . " Zeichen):\n" . $prompt . "\n--- PROMPT END ---", 'PROMPT');
$payload = json_encode([
'contents' => [['parts' => [['text' => $prompt]]]],
'generationConfig' => [
'maxOutputTokens' => 300,
'temperature' => 0.2,
'thinkingConfig' => ['thinkingBudget' => 0], // kein Thinking = alle Token für Antwort
],
]);
$url = 'https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash:generateContent?key=' . urlencode($apiKey);
$resp = '';
$httpCode = 0;
$attempts = [[0, null], [5, 503], [15, 503], [30, 429], [60, 429]];
foreach ($attempts as [$wait, $label]) {
if ($wait > 0) {
logmsg(" Gemini $httpCode warte {$wait}s (retry) ...", 'WARN');
sleep($wait);
}
$ch = curl_init($url);
curl_setopt_array($ch, [
CURLOPT_POST => true,
CURLOPT_RETURNTRANSFER => true,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTPHEADER => ['Content-Type: application/json'],
CURLOPT_POSTFIELDS => $payload,
]);
$resp = curl_exec($ch);
$httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE);
curl_close($ch);
if ($httpCode !== 503 && $httpCode !== 429) break;
}
if ($httpCode !== 200) {
logmsg("Gemini API Fehler HTTP $httpCode: " . substr($resp, 0, 200), 'WARN');
return null;
}
$data = json_decode($resp, true);
$summary = trim($data['candidates'][0]['content']['parts'][0]['text'] ?? '');
if (strlen($summary) < 30) {
logmsg(" Gemini Roh-Antwort: " . substr($resp, 0, 500), 'WARN');
}
return $summary;
}
// ------- Permissions aus main_collection auf Post übertragen -------
function copy_permissions_to_post(int $postId, int $collectionId): void {
$maps = [
['src' => 'main_collection_mandant_link', 'src_col' => 'main_mandant_id', 'dst' => 'knowledgecenter_post_mandant', 'dst_col' => 'mandant_id'],
['src' => 'main_collection_department_link', 'src_col' => 'main_department_id', 'dst' => 'knowledgecenter_post_department', 'dst_col' => 'department_id'],
['src' => 'main_collection_role_link', 'src_col' => 'main_role_id', 'dst' => 'knowledgecenter_post_role', 'dst_col' => 'role_id'],
['src' => 'main_collection_einricht_link', 'src_col' => 'main_einricht_id', 'dst' => 'knowledgecenter_post_einricht', 'dst_col' => 'einricht_id'],
['src' => 'main_collection_fachbereich_link', 'src_col' => 'main_fachbereich_id', 'dst' => 'knowledgecenter_post_fachbereich', 'dst_col' => 'fachbereich_id'],
];
foreach ($maps as $m) {
$sql = "INSERT IGNORE INTO {$m['dst']} (post_id, {$m['dst_col']})
SELECT $postId, {$m['src_col']} FROM {$m['src']} WHERE main_collection_id = $collectionId";
mysqli_query($GLOBALS['mysql_con'], $sql);
}
}
// ------- Artikel laden -------
$whereClause = "mc.main_collection_setup_id = 3 AND mc.is_archived = 0 AND mc.active = 1 AND mc.description != ''";
if ($opts['collection-id'] !== null) {
$whereClause .= " AND mc.id = " . (int)$opts['collection-id'];
}
// ------- Summaries-only Modus: fehlende Summaries für bereits migrierte Posts nachfüllen -------
if ($opts['summaries-only']) {
if (!$apiKey) { fwrite(STDERR, "--summaries-only benötigt --api-key\n"); exit(2); }
$sumSql = "SELECT p.id as post_id, p.legacy_collection_id, pv.id as version_id, pv.title,
pv.content
FROM knowledgecenter_posts p
JOIN knowledgecenter_post_versions pv ON pv.post_id = p.id AND pv.version_number = 1
WHERE (pv.summary IS NULL OR pv.summary = '')
AND p.legacy_collection_id IS NOT NULL";
if ($opts['collection-id'] !== null) {
$sumSql .= " AND p.legacy_collection_id = " . (int)$opts['collection-id'];
}
if ($opts['limit'] !== null) {
$sumSql .= " LIMIT " . (int)$opts['limit'];
}
$sumResult = mysqli_query($GLOBALS['mysql_con'], $sumSql);
$sumTotal = mysqli_num_rows($sumResult);
logmsg("Summaries-only: $sumTotal Posts ohne Summary", 'HEAD');
$sumStats = ['ok' => 0, 'fail' => 0];
while ($sumRow = mysqli_fetch_assoc($sumResult)) {
$postId = (int)$sumRow['post_id'];
$collId = (int)$sumRow['legacy_collection_id'];
$versionId = (int)$sumRow['version_id'];
$title = $sumRow['title'];
$files = get_article_files($collId);
$summary = generate_summary($sumRow['content'], $files, $userdataDir, $apiKey, $title) ?? '';
if ($summary === '') {
logmsg(" [FAIL] Post $postId \"$title\": leere Summary", 'WARN');
$sumStats['fail']++;
} else {
$sumEsc = mysqli_real_escape_string($GLOBALS['mysql_con'], $summary);
mysqli_query($GLOBALS['mysql_con'],
"UPDATE knowledgecenter_post_versions SET summary = '$sumEsc' WHERE id = $versionId");
logmsg("[OK ] Post $postId \"$title\": " . mb_substr($summary, 0, 80), 'OK');
$sumStats['ok']++;
}
usleep(200000);
}
logmsg(sprintf("Summaries fertig. ok=%d fail=%d", $sumStats['ok'], $sumStats['fail']), 'HEAD');
logmsg("Log-Datei: $logFile", 'HEAD');
fclose($logFp);
exit(0);
}
$sql = "SELECT mc.id, mc.description, mc.creation_date, mc.modified_date,
mc.validity_from, mc.validity_to
FROM main_collection mc
WHERE $whereClause
ORDER BY mc.id";
if ($opts['limit'] !== null) {
$sql .= " LIMIT " . (int)$opts['limit'];
}
$result = mysqli_query($GLOBALS['mysql_con'], $sql);
$total = mysqli_num_rows($result);
logmsg("Modus=" . ($dryRun ? 'DRY-RUN' : 'APPLY') . " Artikel=$total Limit=" .
($opts['limit'] ?? 'alle') . " API=" . ($apiKey && !$skipSummary ? 'ja' : 'nein'), 'HEAD');
$stats = ['ok' => 0, 'skipped' => 0, 'fail' => 0];
while ($row = mysqli_fetch_assoc($result)) {
$cid = (int)$row['id'];
$title = $row['description'];
// Idempotenz-Check
$already = mysqli_query($GLOBALS['mysql_con'],
"SELECT id FROM knowledgecenter_posts WHERE legacy_collection_id = $cid LIMIT 1");
if ($already && mysqli_num_rows($already) > 0) {
logmsg("Artikel $cid \"$title\" bereits migriert, überspringe.", 'SKIP');
$stats['skipped']++;
continue;
}
$parts = assemble_content($cid);
$teaserHtml = $parts['teaser'];
$htmlContent = $parts['content'];
$files = get_article_files($cid);
$htmlLen = strlen($htmlContent);
$catRes = mysqli_query($GLOBALS['mysql_con'],
"SELECT knowledgecenter_category_id FROM knowledgecenter_collection_link WHERE main_collection_id = $cid");
$catIds = [];
while ($cr = mysqli_fetch_assoc($catRes)) {
$catIds[] = (int)$cr['knowledgecenter_category_id'];
}
logmsg("Artikel $cid \"$title\" | Teaser=" . strlen($teaserHtml) . "B | HTML=" . $htmlLen . "B | Dateien=" . count($files) .
" | Kategorien=[" . implode(',', $catIds) . "]");
if ($dryRun) {
$stats['ok']++;
continue;
}
// Summary generieren (vor der Transaktion, API-Call außerhalb)
$summary = '';
if (!$skipSummary && $apiKey) {
$fullContent = $teaserHtml . $htmlContent;
$generated = generate_summary($fullContent, $files, $userdataDir, $apiKey, $title);
$summary = $generated ?? '';
logmsg(" Summary: " . (strlen($summary) > 0 ? '"' . mb_substr($summary, 0, 100) . '"' : '(leer)'));
usleep(200000); // 200ms Pause für API rate limiting
}
// HTML-Entities dekodieren für saubere UTF-8 Speicherung
$cleanHtml = html_entity_decode($htmlContent, ENT_HTML5 | ENT_QUOTES, 'UTF-8');
mysqli_query($GLOBALS['mysql_con'], 'START TRANSACTION');
try {
$titleEsc = mysqli_real_escape_string($GLOBALS['mysql_con'], $title);
$summaryEsc = mysqli_real_escape_string($GLOBALS['mysql_con'], $summary);
$teaserEsc = mysqli_real_escape_string($GLOBALS['mysql_con'], $teaserHtml);
$contentEsc = mysqli_real_escape_string($GLOBALS['mysql_con'], $cleanHtml);
$createdAt = date('Y-m-d H:i:s', (int)$row['creation_date']);
$modifiedAt = date('Y-m-d H:i:s', (int)$row['modified_date']);
$validFrom = $row['validity_from'] ? "'" . $row['validity_from'] . "'" : 'NULL';
$validUntil = $row['validity_to'] ? "'" . $row['validity_to'] . "'" : 'NULL';
// knowledgecenter_posts
$sqlPost = "INSERT INTO knowledgecenter_posts (legacy_collection_id, created_at, modified_at)
VALUES ($cid, '$createdAt', '$modifiedAt')";
if (!mysqli_query($GLOBALS['mysql_con'], $sqlPost)) {
throw new RuntimeException("Post-Insert: " . mysqli_error($GLOBALS['mysql_con']));
}
$postId = (int)mysqli_insert_id($GLOBALS['mysql_con']);
// knowledgecenter_post_versions
$sqlVer = "INSERT INTO knowledgecenter_post_versions
(post_id, version_number, language_id, title, teaser, content, summary, state, modified_at, valid_from, valid_until)
VALUES
($postId, 1, 1, '$titleEsc', '$teaserEsc', '$contentEsc', '$summaryEsc', 1, '$modifiedAt', $validFrom, $validUntil)";
if (!mysqli_query($GLOBALS['mysql_con'], $sqlVer)) {
throw new RuntimeException("Version-Insert: " . mysqli_error($GLOBALS['mysql_con']));
}
// Kategorie-Links
foreach ($catIds as $catId) {
mysqli_query($GLOBALS['mysql_con'],
"INSERT IGNORE INTO knowledgecenter_category_post (category_id, post_id) VALUES ($catId, $postId)");
}
// Permissions aus main_collection auf Post übertragen
copy_permissions_to_post($postId, $cid);
// Dateiverknüpfungen: Original-Pfad beibehalten, keine Kopie
foreach ($files as $file) {
$srcPath = $userdataDir . '/' . $file['filename'];
if (!file_exists($srcPath)) {
logmsg(" Quelldatei nicht gefunden: " . $file['filename'], 'WARN');
continue;
}
$relPath = '/userdata/' . $file['filename'];
$descEsc = mysqli_real_escape_string($GLOBALS['mysql_con'], $file['description'] ?: basename($file['filename']));
$pathEsc = mysqli_real_escape_string($GLOBALS['mysql_con'], $relPath);
mysqli_query($GLOBALS['mysql_con'],
"INSERT INTO knowledgecenter_post_files (post_id, description, path)
VALUES ($postId, '$descEsc', '$pathEsc')");
logmsg(" Datei verknüpft: " . $file['filename']);
}
mysqli_query($GLOBALS['mysql_con'], 'COMMIT');
logmsg("[OK ] Artikel $cid → Post $postId \"$title\"", 'OK');
$stats['ok']++;
} catch (\Throwable $e) {
mysqli_query($GLOBALS['mysql_con'], 'ROLLBACK');
logmsg("[FAIL] Artikel $cid \"$title\": " . $e->getMessage(), 'FAIL');
$stats['fail']++;
}
}
logmsg(sprintf("Fertig. ok=%d skipped=%d fail=%d modus=%s",
$stats['ok'], $stats['skipped'], $stats['fail'],
$dryRun ? 'DRY-RUN' : 'APPLY'), 'HEAD');
logmsg("Log-Datei: $logFile", 'HEAD');
fclose($logFp);