Skip to content

Commit

Permalink
Merge remote-tracking branch 'upstream/master'
Browse files Browse the repository at this point in the history
  • Loading branch information
@labster
labster committed Jun 3, 2024
2 parents 3eedd54 + b2c4db8 commit be81031
Show file tree
Hide file tree
Showing 4 changed files with 146 additions and 236 deletions.
9 changes: 1 addition & 8 deletions extension.json
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "SemanticDependencyUpdater",
"version": "3.0.1",
"version": "3.0.3",
"author": [
"Simon Heimler (gesinn.it GmbH & Co. KG)",
"Alexander Gesinn (gesinn.it GmbH & Co. KG)",
Expand All @@ -23,15 +23,8 @@
"Hooks": {
"SMW::SQLStore::AfterDataUpdateComplete": [
"SDU\\Hooks::onAfterDataUpdateComplete"
],
"PageDelete": [
"SDU\\Hooks::onPageDelete"
]
},
"JobClasses": {
"RebuildDataJob": "SDU\\RebuildDataJob",
"PageUpdaterJob": "SDU\\PageUpdaterJob"
},
"callback": "SDU\\Hooks::setup",
"config": {
"SDUProperty": "Semantic Dependency",
Expand Down
299 changes: 145 additions & 154 deletions src/Hooks.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,200 +4,191 @@

use DeferredUpdates;
use JobQueueGroup;
use MediaWiki\MediaWikiServices;
use MediaWiki\Page\WikiPageFactory;
use SMW\Options;
use SMW\MediaWiki\Jobs\UpdateJob;
use SMW\Services\ServicesFactory as ApplicationFactory;
use SMWDIBlob;
use SMWDIWikiPage;
use SMWQueryProcessor;
use SMWSemanticData;
use SMWStore;
use Title;
use WikiPage;

class Hooks {

public static function setup() {
if ( !defined( 'MEDIAWIKI' ) ) {
die();
}

if ( !defined( 'SMW_VERSION' ) ) {
die( "ERROR: Semantic MediaWiki must be installed for Semantic Dependency Updater to run!" );
}
public static function setup() {
if ( !defined( 'MEDIAWIKI' ) ) {
die();
}

// Note: at the time SMW::SQLStore::BeforeDeleteSubjectComplete fires there is no data already
// so the PageDelete hook is used
public static function onPageDelete( $page, $deleter, string $reason, $status, bool $suppress ) {
$store = smwfGetStore();
$diWikiPage = SMWDIWikiPage::newFromTitle( Title::newFromDBkey( $page->getDBkey() ) );
$smwData = $store->getSemanticData( $diWikiPage );
self::onAfterDataUpdateComplete( $store, $smwData, null );
if ( !defined( 'SMW_VERSION' ) ) {
die( "ERROR: Semantic MediaWiki must be installed for Semantic Dependency Updater to run!" );
}
}

public static function onAfterDataUpdateComplete(
SMWStore $store, SMWSemanticData $newData,
$compositePropertyTableDiffIterator
) {
global $wgSDUProperty;
global $wgSDUTraversed;
public static function onAfterDataUpdateComplete(
SMWStore $store, SMWSemanticData $newData,
$compositePropertyTableDiffIterator
) {
global $wgSDUProperty;
global $wgSDUTraversed;

if ( !isset( $wgSDUTraversed ) ) {
$wgSDUTraversed = [];
}

$wgSDUProperty = str_replace( ' ', '_', $wgSDUProperty );
$subject = $newData->getSubject();
$title = $subject->getTitle();
if ( $title == null ) {
return true;
}
if ( !isset( $wgSDUTraversed ) ) {
$wgSDUTraversed = [];
}

$id = $title->getPrefixedDBKey();
$wgSDUProperty = str_replace( ' ', '_', $wgSDUProperty );
$subject = $newData->getSubject();
$title = $subject->getTitle();
if ( $title == null ) {
return true;
}

wfDebugLog( 'SemanticDependencyUpdater', "[SDU] --> " . $title );
$id = $title->getPrefixedDBKey();

// FIRST CHECK: Does the page data contain a $wgSDUProperty semantic property ?
$properties = $newData->getProperties();
if ( !isset( $properties[$wgSDUProperty] ) ) {
wfDebugLog( 'SemanticDependencyUpdater', "[SDU] <-- No SDU property found" );
return true;
}
wfDebugLog( 'SemanticDependencyUpdater', "[SDU] --> " . $title );

if ( $compositePropertyTableDiffIterator !== null ) {
$diffTable = $compositePropertyTableDiffIterator->getOrderedDiffByTable();
// FIRST CHECK: Does the page data contain a $wgSDUProperty semantic property ?
$properties = $newData->getProperties();
if ( !isset( $properties[$wgSDUProperty] ) ) {
wfDebugLog( 'SemanticDependencyUpdater', "[SDU] <-- No SDU property found" );
return true;
}

// SECOND CHECK: Have there been actual changes in the data? (Ignore internal SMW data!)
// TODO: Introduce an explicit list of Semantic Properties to watch ?
unset( $diffTable['smw_fpt_mdat'] ); // Ignore SMW's internal properties "smw_fpt_mdat"
$diffTable = $compositePropertyTableDiffIterator->getOrderedDiffByTable();
$smwSID = $compositePropertyTableDiffIterator->getSubject()->getId();
// SECOND CHECK: Have there been actual changes in the data? (Ignore internal SMW data!)
// TODO: Introduce an explicit list of Semantic Properties to watch ?
unset( $diffTable['smw_fpt_mdat'] ); // Ignore SMW's internal properties "smw_fpt_mdat"

// lets try first to check the data tables: https://www.semantic-mediawiki.org/wiki/Help:Database_schema
// if change, on pageID from Issue, that is not REvision ID, then trigger all changes
$triggerSemanticDependencies = false;

if ( count( $diffTable ) > 0 ) {
wfDebugLog( 'SemanticDependencyUpdater', "[SDU] -----> Data changes detected" );

foreach ( $diffTable as $key => $value ) {
if ( strpos( $key, 'smw_di' ) === 0 && is_array( $value ) ) {
foreach ( $value["insert"] as $insert ) {
if ( $insert["s_id"] == $smwSID ) {
if ( $insert["p_id"] != 506 ) {
$triggerSemanticDependencies = true;
break 2;
} // revision ID change is good, but must not trigger UpdateJob for semantic dependencies
}
}
}
}
} else {
wfDebugLog( 'SemanticDependencyUpdater', "[SDU] <-- No semantic data changes detected" );
return true;
}

if ( count( $diffTable ) > 0 ) {
// wfDebugLog('SemanticDependencyUpdater', "[SDU] diffTable: " . print_r($diffTable, true));
wfDebugLog( 'SemanticDependencyUpdater', "[SDU] -----> Data changes detected" );
} else {
wfDebugLog( 'SemanticDependencyUpdater', "[SDU] <-- No semantic data changes detected" );
// THIRD CHECK: Has this page been already traversed more than twice?
// This should only be the case when SMW errors occur.
// In that case, the diffTable contains everything and SDU can't know if changes happened
if ( array_key_exists( $id, $wgSDUTraversed ) ) {
$wgSDUTraversed[$id] = $wgSDUTraversed[$id] + 1;
} else {
$wgSDUTraversed[$id] = 1;
}
if ( $wgSDUTraversed[$id] > 2 ) {
wfDebugLog( 'SemanticDependencyUpdater', "[SDU] <-- Already traversed" );
return true;
}

return true;
// QUERY AND UPDATE DEPENDENCIES

// SMW\SemanticData $newData
// SMWDataItem[] $dataItem
$wikiPageValues = [];
if ( $triggerSemanticDependencies ) {
$dataItem = $newData->getPropertyValues( $properties[$wgSDUProperty] );
if ( $dataItem != null ) {
foreach ( $dataItem as $valueItem ) {
if ( $valueItem instanceof SMWDIBlob && $valueItem->getString() != $id ) {
$wikiPageValues = array_merge( $wikiPageValues, self::updatePagesMatchingQuery( $valueItem->getSerialization() ) );
}
}
}
} else {
$wikiPageValues = [ $subject ];
}

// THIRD CHECK: Has this page been already traversed more than twice?
// This should only be the case when SMW errors occur.
// In that case, the diffTable contains everything and SDU can't know if changes happened
if ( array_key_exists( $id, $wgSDUTraversed ) ) {
$wgSDUTraversed[$id] = $wgSDUTraversed[$id] + 1;
} else {
$wgSDUTraversed[$id] = 1;
}
if ( $wgSDUTraversed[$id] > 2 ) {
wfDebugLog( 'SemanticDependencyUpdater', "[SDU] <-- Already traversed" );
return true;
}
self::rebuildData( $triggerSemanticDependencies, $wikiPageValues, $subject );

// QUERY AND UPDATE DEPENDENCIES
return true;
}

// SMW\SemanticData $newData
// SMWDataItem[] $dataItem
$dataItem = $newData->getPropertyValues( $properties[$wgSDUProperty] );
/**
* @param string $queryString Query string, excluding [[ and ]] brackets
*/
private static function updatePagesMatchingQuery( $queryString ) {
global $sfgListSeparator;

$wikiPageValues = [];
if ( $dataItem != null ) {
foreach ( $dataItem as $valueItem ) {
if ( $valueItem instanceof SMWDIBlob ) {
$wikiPageValues = array_merge( $wikiPageValues, self::updatePagesMatchingQuery( $valueItem->getSerialization() ) );
}
}
}
$queryString = str_replace( 'AND', ']] [[', $queryString );
$queryString = str_replace( 'OR', ']] OR [[', $queryString );

self::rebuildData( $wikiPageValues, $store );
return true;
// If SF is installed, get the separator character and change it into ||
// Otherwise SDU won't work with multi-value properties
if ( isset( $sfgListSeparator ) ) {
$queryString = rtrim( $queryString, $sfgListSeparator );
$queryString = str_replace( $sfgListSeparator, ' || ', $queryString );
}

/**
* @param string $queryString Query string, excluding [[ and ]] brackets
*/
private static function updatePagesMatchingQuery( $queryString ) {
global $sfgListSeparator;
wfDebugLog( 'SemanticDependencyUpdater', "[SDU] --------> [[$queryString]]" );

$queryString = str_replace( 'AND', ']] [[', $queryString );
$queryString = str_replace( 'OR', ']] OR [[', $queryString );
$store = smwfGetStore();

// If SF is installed, get the separator character and change it into ||
// Otherwise SDU won't work with multi-value properties
if ( isset( $sfgListSeparator ) ) {
$queryString = rtrim( $queryString, $sfgListSeparator );
$queryString = str_replace( $sfgListSeparator, ' || ', $queryString );
}
$params = [
'limit' => 10000,
];
$processedParams = SMWQueryProcessor::getProcessedParams( $params );
$query =
SMWQueryProcessor::createQuery( "[[$queryString]]", $processedParams, SMWQueryProcessor::SPECIAL_PAGE );
$result = $store->getQueryResult( $query );
$wikiPageValues = $result->getResults();

wfDebugLog( 'SemanticDependencyUpdater', "[SDU] --------> [[$queryString]]" );
return $wikiPageValues;
}

$store = smwfGetStore();
/**
* Rebuilds data of the given wikipages to regenerate semantic attrubutes and re-run queries
*/
public static function rebuildData( $triggerSemanticDependencies, $wikiPageValues, $subject ) {
global $wgSDUUseJobQueue;

$params = [
'limit' => 10000,
];
$processedParams = SMWQueryProcessor::getProcessedParams( $params );
$query =
SMWQueryProcessor::createQuery( "[[$queryString]]", $processedParams, SMWQueryProcessor::SPECIAL_PAGE );
$result = $store->getQueryResult( $query ); // SMWQueryResult
$wikiPageValues = $result->getResults(); // array of SMWWikiPageValues
if ( $wgSDUUseJobQueue ) {
$jobFactory = ApplicationFactory::getInstance()->newJobFactory();

return $wikiPageValues;
}
if ( $triggerSemanticDependencies ) {

/**
* Rebuilds data of the given wikipages to regenerate semantic attrubutes and re-run queries
*
* @param SMWWikiPageValues[] $wikiPageValues
* @param SMWStore $store
*/
public static function rebuildData( $wikiPageValues, $store ) {
global $wgSDUUseJobQueue;
$jobs = [];

$pageArray = [];
foreach ( $wikiPageValues as $wikiPageValue ) {
$page = MediaWikiServices::getInstance()->getWikiPageFactory()->newFromID( $wikiPageValue->getTitle()->getArticleId() );
if ( $page ) {
$pageArray[] = $page->getTitle()->getPrefixedText();
}
$jobs[] = $jobFactory->newUpdateJob(
$wikiPageValue->getTitle(),
[
UpdateJob::FORCED_UPDATE => true,
'shallowUpdate' => false
]
);
}
$pageString = implode( "|", $pageArray );

// TODO: A threshold when to switch to Queue Jobs might be smarter

if ( $pageString !== "" ) {
if ( $wgSDUUseJobQueue ) {
$jobs = [];
$jobs[] = new RebuildDataJob( [
'pageString' => $pageString,
] );
foreach ( $wikiPageValues as $page ) {
$jobs[] = new PageUpdaterJob( [
'page' => $page
] );
}
if ( $jobs ) {
JobQueueGroup::singleton()->lazyPush( $jobs );
}
} else {
DeferredUpdates::addCallableUpdate( static function () use ( $store, $pageString ) {
wfDebugLog( 'SemanticDependencyUpdater', "[SDU] --------> [rebuildData] $pageString" );
$maintenanceFactory = ApplicationFactory::getInstance()->newMaintenanceFactory();

$dataRebuilder = $maintenanceFactory->newDataRebuilder( $store );
$dataRebuilder->setOptions(
new Options( [ 'page' => $pageString ] )
);
$dataRebuilder->rebuild();
foreach ( explode( '|', $pageString ) as $wikipage ) {
$wikipage = new WikiPage( Title::newFromText( $pageString ) );
$wikipage->doPurge();
}
} );
}
if ( $jobs ) {
JobQueueGroup::singleton()->lazyPush( $jobs );
}
} else {
DeferredUpdates::addCallableUpdate( static function () use ( $jobFactory, $wikiPageValues ) {
$job = $jobFactory->newUpdateJob(
$wikiPageValues[0]->getTitle(),
[
UpdateJob::FORCED_UPDATE => true,
'shallowUpdate' => false
]
);
$job->run();
} );
}

}
}

}
Loading

0 comments on commit be81031

Please sign in to comment.