Page MenuHomePhabricator
Paste P93891

WIP - fixes to VisualData
ActivePublic
Actions

Authored by freephile on Jun 5 2026, 2:23 PM.
Project Tags
None
Referenced Files
F86767606: WIP - fixes to VisualData
Jun 5 2026, 2:37 PM
F86765864: WIP - fixes to VisualData
Jun 5 2026, 2:23 PM
Subscribers
Thomas-topway-it
## Patch
diff --git a/includes/classes/ResultPrinter.php b/includes/classes/ResultPrinter.php
index 2a1c9f6..960e79a 100644
--- a/includes/classes/ResultPrinter.php
+++ b/includes/classes/ResultPrinter.php
@@ -603,8 +603,10 @@ class ResultPrinter {
}
if ( $this->isHtml() && $isRoot ) {
- return Parser::stripOuterParagraph(
- $this->parser->recursiveTagParseFully( $ret )
+ return $this->stripParsoidEditSectionMarkers(
+ Parser::stripOuterParagraph(
+ $this->parser->recursiveTagParseFully( $ret )
+ )
);
}
@@ -665,6 +667,22 @@ class ResultPrinter {
$replaceAlias( self::$titleAliases, $title->getFullText() );
$replaceAlias( self::$categoriesAliases, implode( ', ', $categories ) );
+ return $this->stripParsoidEditSectionMarkers( $value );
+ }
+
+ /**
+ * Remove parser-only edit section markers from rendered HTML fragments.
+ *
+ * @param string $value
+ * @return string
+ */
+ protected function stripParsoidEditSectionMarkers( $value ) {
+ if ( strpos( $value, '<mw:editsection' ) === false ) {
+ return $value;
+ }
+
+ $value = preg_replace( '#<mw:editsection\b[^>]*>.*?</mw:editsection>#is', '', $value );
+ $value = preg_replace( '#<mw:editsection\b[^>]*/\s*>#is', '', $value );
return $value;
}
diff --git a/includes/classes/SubmitForm.php b/includes/classes/SubmitForm.php
index 55757fd..e59fb20 100644
--- a/includes/classes/SubmitForm.php
+++ b/includes/classes/SubmitForm.php
@@ -139,14 +139,30 @@ class SubmitForm {
private function parseWikitext( $value ) {
// return $this->parser->recursiveTagParseFully( $str );
if ( !is_array( $value ) ) {
- return Parser::stripOuterParagraph( $this->output->parseAsContent( $value ) );
+ return $this->stripParsoidEditSectionMarkers(
+ Parser::stripOuterParagraph( $this->output->parseAsContent( $value ) )
+ );
}
$self = $this;
return array_map( static function ( $v ) use ( $self ) {
- return Parser::stripOuterParagraph( $self->output->parseAsContent( $v ) );
+ return $self->stripParsoidEditSectionMarkers(
+ Parser::stripOuterParagraph( $self->output->parseAsContent( $v ) )
+ );
}, $value );
}
+ /**
+ * Prevent parser-only edit section markers from being persisted in VisualData values.
+ *
+ * @param string $html
+ * @return string
+ */
+ private function stripParsoidEditSectionMarkers( $html ) {
+ $html = preg_replace( '#<mw:editsection\b[^>]*>.*?</mw:editsection>#is', '', $html );
+ $html = preg_replace( '#<mw:editsection\b[^>]*/\s*>#is', '', $html );
+ return $html;
+ }
+
/**
* @param string $filename
* @param string $filekey
diff --git a/resources/VisualDataDatatables.js b/resources/VisualDataDatatables.js
index 902a4b0..175c925 100644
--- a/resources/VisualDataDatatables.js
+++ b/resources/VisualDataDatatables.js
@@ -305,6 +305,16 @@ const VisualDataDatatables = function ( el, elIndex ) {
} );
}
+ var stripEditSectionMarkers = function ( value ) {
+ if ( typeof value !== 'string' || value.indexOf( '<mw:editsection' ) === -1 ) {
+ return value;
+ }
+
+ return value
+ .replace( /<mw:editsection\b[^>]*>[\s\S]*?<\/mw:editsection>/gi, '' )
+ .replace( /<mw:editsection\b[^>]*\/\s*>/gi, '' );
+ };
+
var initColumnSort = function ( order, headers ) {
var ret = [];
// eg. new_property asc, new_property_2 desc
@@ -751,6 +761,10 @@ html-num-fmt
// },
render: function ( thisData, type, row, meta ) {
+ thisData = thisData.map( function ( value ) {
+ return stripEditSectionMarkers( value );
+ } );
+
// or use mapPathSchema[ printout ].format
if (
meta.col in mapColumnIndexFormat &&
diff --git a/resources/VisualDataForms.js b/resources/VisualDataForms.js
index 0744c5b..6facaba 100644
--- a/resources/VisualDataForms.js
+++ b/resources/VisualDataForms.js
@@ -157,6 +157,52 @@ const VisualDataForms = function ( El, Config, Form, FormIndex, Schemas, WindowM
);
}
+ function isBadTokenError( res ) {
+ if ( !res ) {
+ return false;
+ }
+
+ if ( typeof res === 'string' ) {
+ return res.toLowerCase().indexOf( 'badtoken' ) !== -1;
+ }
+
+ if ( res.error && res.error.code === 'badtoken' ) {
+ return true;
+ }
+
+ if ( res.xhr && res.xhr.responseJSON &&
+ res.xhr.responseJSON.error &&
+ res.xhr.responseJSON.error.code === 'badtoken'
+ ) {
+ return true;
+ }
+
+ return false;
+ }
+
+ function postWithTokenRetry( payload ) {
+ return new Promise( function ( resolve, reject ) {
+ var api = new mw.Api();
+
+ api.postWithToken( 'csrf', payload )
+ .done( resolve )
+ .fail( function ( err ) {
+ if ( !isBadTokenError( err ) ) {
+ reject( err );
+ return;
+ }
+
+ api.getToken( 'csrf' )
+ .done( function ( token ) {
+ api.post( $.extend( {}, payload, { token: token } ) )
+ .done( resolve )
+ .fail( reject );
+ } )
+ .fail( reject );
+ } );
+ } );
+ }
+
function callbackShowError( schemaName, errorMessage, errors, hiddenErrors ) {
// remove previous error messages
@@ -3099,9 +3145,8 @@ const VisualDataForms = function ( El, Config, Form, FormIndex, Schemas, WindowM
};
return new Promise( ( resolve, reject ) => {
- new mw.Api()
- .postWithToken( 'csrf', payload )
- .done( function ( thisRes ) {
+ postWithTokenRetry( payload )
+ .then( function ( thisRes ) {
resolve();
if ( payload.action in thisRes ) {
var data = JSON.parse( thisRes[ payload.action ].result );
@@ -3124,7 +3169,7 @@ const VisualDataForms = function ( El, Config, Form, FormIndex, Schemas, WindowM
}
}
} )
- .fail( function ( thisRes ) {
+ .catch( function ( thisRes ) {
// eslint-disable-next-line no-console
console.error( 'visualdata-submit-form', thisRes );
reject( thisRes );
## New File
diff --git a/maintenance/RenameSchemaProperty.php b/maintenance/RenameSchemaProperty.php
new file mode 100644
index 0000000..c7a7438
--- /dev/null
+++ b/maintenance/RenameSchemaProperty.php
@@ -0,0 +1,272 @@
+<?php
+
+/**
+ * This file is part of the MediaWiki extension VisualData.
+ *
+ * VisualData is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * VisualData is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with VisualData. If not, see <http://www.gnu.org/licenses/>.
+ *
+ * @file
+ * @ingroup extensions
+ */
+
+use MediaWiki\Revision\SlotRecord;
+use MediaWiki\Context\RequestContext;
+use MediaWiki\Extension\VisualData\DatabaseManager;
+use MediaWiki\MediaWikiServices;
+
+$IP = getenv( 'MW_INSTALL_PATH' );
+if ( $IP === false ) {
+ $IP = __DIR__ . '/../../..';
+}
+require_once "$IP/maintenance/Maintenance.php";
+
+class RenameSchemaProperty extends Maintenance {
+ /** @var User */
+ private $user;
+
+ /** @var DatabaseManager */
+ private $databaseManager;
+
+ public function __construct() {
+ parent::__construct();
+ $this->addDescription( 'Rename a property key in VisualData stored records for a schema and rebuild mirrored query data.' );
+ $this->requireExtension( 'VisualData' );
+
+ $this->addOption( 'schema', 'Schema name to process.', false, true );
+ $this->addOption( 'old-key', 'Existing property key to rename.', false, true );
+ $this->addOption( 'new-key', 'New property key.', false, true );
+ $this->addOption( 'prefix', 'Optional title prefix filter (for example: Tasks/).', false, true );
+ $this->addOption( 'limit', 'Optional maximum number of pages to process.', false, true );
+ $this->addOption( 'force', 'Overwrite new-key if it already exists.', false, false );
+ $this->addOption( 'dry-run', 'Show what would change without writing revisions.', false, false );
+ }
+
+ /**
+ * @inheritDoc
+ */
+ public function execute() {
+ $schemaName = $this->getOption( 'schema', 'Task' );
+ $oldKey = $this->getOption( 'old-key', 'comment' );
+ $newKey = $this->getOption( 'new-key', 'task_detail' );
+ $titlePrefix = $this->getOption( 'prefix', '' );
+ $limit = (int)$this->getOption( 'limit', 0 );
+ $force = (bool)$this->getOption( 'force', false );
+ $dryRun = (bool)$this->getOption( 'dry-run', false );
+
+ if ( $oldKey === $newKey ) {
+ $this->fatalError( '--old-key and --new-key must be different.' );
+ }
+
+ $this->user = User::newSystemUser( 'Maintenance script', [ 'steal' => true ] );
+ $this->databaseManager = new DatabaseManager();
+
+ $titles = $this->databaseManager->pagesWithSchema( $schemaName );
+
+ if ( $titlePrefix !== '' ) {
+ $titles = array_values( array_filter( $titles, static function ( $title ) use ( $titlePrefix ) {
+ return strpos( $title->getPrefixedText(), $titlePrefix ) === 0;
+ } ) );
+ }
+
+ if ( $limit > 0 ) {
+ $titles = array_slice( $titles, 0, $limit );
+ }
+
+ $total = count( $titles );
+ $this->output( "Found {$total} pages for schema '{$schemaName}'.\n" );
+
+ $updated = 0;
+ $skippedNoSchema = 0;
+ $skippedNoOldKey = 0;
+ $skippedConflict = 0;
+ $errorsCount = 0;
+
+ foreach ( $titles as $title ) {
+ $titleText = $title->getPrefixedText();
+ $jsonData = \VisualData::getJsonData( $title );
+
+ if ( !$jsonData || empty( $jsonData['schemas'] ) || !isset( $jsonData['schemas'][$schemaName] ) ) {
+ $skippedNoSchema++;
+ $this->output( "SKIP {$titleText}: schema payload not found in jsondata.\n" );
+ continue;
+ }
+
+ $schemaPayload = $jsonData['schemas'][$schemaName];
+ if ( !is_array( $schemaPayload ) || !array_key_exists( $oldKey, $schemaPayload ) ) {
+ $skippedNoOldKey++;
+ $this->output( "SKIP {$titleText}: key '{$oldKey}' not present.\n" );
+ continue;
+ }
+
+ if ( array_key_exists( $newKey, $schemaPayload ) && !$force ) {
+ $skippedConflict++;
+ $this->output( "SKIP {$titleText}: key '{$newKey}' already exists (use --force to overwrite).\n" );
+ continue;
+ }
+
+ $newValue = $schemaPayload[$oldKey];
+ $jsonData['schemas'][$schemaName][$newKey] = $newValue;
+ unset( $jsonData['schemas'][$schemaName][$oldKey] );
+
+ // Keep raw untransformed key paths aligned when present.
+ $this->migrateUntransformedPath( $jsonData, $schemaName, $oldKey, $newKey, $force );
+
+ if ( $dryRun ) {
+ $updated++;
+ $this->output( "DRY-RUN {$titleText}: would rename {$schemaName}.{$oldKey} -> {$schemaName}.{$newKey}.\n" );
+ continue;
+ }
+
+ $errors = [];
+ if ( !$this->saveJsonDataInternal( $title, $jsonData, $errors ) ) {
+ $errorsCount++;
+ $this->output( "ERROR {$titleText}: internal slot update failed.\n" );
+ if ( count( $errors ) ) {
+ $this->output( ' - ' . implode( ' | ', $errors ) . "\n" );
+ }
+ continue;
+ }
+
+ $errors = [];
+ $context = RequestContext::newExtraneousContext( $title );
+ $rebuildStatus = $this->rebuildArticleDataInternal( $context, $title, $jsonData, $errors );
+ if ( $rebuildStatus === false ) {
+ $errorsCount++;
+ $this->output( "ERROR {$titleText}: rebuildArticleData failed.\n" );
+ if ( count( $errors ) ) {
+ $this->output( ' - ' . implode( ' | ', $errors ) . "\n" );
+ }
+ continue;
+ }
+
+ $updated++;
+ $this->output( "UPDATED {$titleText}: {$schemaName}.{$oldKey} -> {$schemaName}.{$newKey}.\n" );
+ }
+
+ $this->output( "\nSummary:\n" );
+ $this->output( " Updated: {$updated}\n" );
+ $this->output( " Skipped (no schema payload): {$skippedNoSchema}\n" );
+ $this->output( " Skipped (missing old key): {$skippedNoOldKey}\n" );
+ $this->output( " Skipped (new key conflict): {$skippedConflict}\n" );
+ $this->output( " Errors: {$errorsCount}\n" );
+
+ if ( $dryRun ) {
+ $this->output( "\nDry run mode was enabled. No pages were modified.\n" );
+ }
+ }
+
+ /**
+ * @param array &$jsonData
+ * @param string $schemaName
+ * @param string $oldKey
+ * @param string $newKey
+ * @param bool $force
+ * @return void
+ */
+ private function migrateUntransformedPath( array &$jsonData, $schemaName, $oldKey, $newKey, $force ) {
+ if ( empty( $jsonData['schemas-data'] )
+ || empty( $jsonData['schemas-data']['untransformed'] )
+ || !is_array( $jsonData['schemas-data']['untransformed'] )
+ ) {
+ return;
+ }
+
+ $oldPath = $schemaName . '/' . $oldKey;
+ $newPath = $schemaName . '/' . $newKey;
+
+ if ( !array_key_exists( $oldPath, $jsonData['schemas-data']['untransformed'] ) ) {
+ return;
+ }
+
+ if ( array_key_exists( $newPath, $jsonData['schemas-data']['untransformed'] ) && !$force ) {
+ return;
+ }
+
+ $jsonData['schemas-data']['untransformed'][$newPath] = $jsonData['schemas-data']['untransformed'][$oldPath];
+ unset( $jsonData['schemas-data']['untransformed'][$oldPath] );
+ }
+
+ /**
+ * Save the updated jsondata slot using the internal edit path.
+ *
+ * @param Title $title
+ * @param array $jsonData
+ * @param array &$errors
+ * @return bool
+ */
+ private function saveJsonDataInternal( $title, array $jsonData, array &$errors ) {
+ $wikiPage = \VisualData::getWikiPage( $title );
+ if ( !$wikiPage ) {
+ $errors[] = 'no wiki page';
+ return false;
+ }
+
+ $slots = $wikiPage->getRevisionRecord() ? $wikiPage->getRevisionRecord()->getSlots()->getSlots() : [];
+ $targetSlot = \VisualData::getTargetSlot( $title, SlotRecord::MAIN );
+ $slotContent = ContentHandler::makeContent( json_encode( $jsonData ), $title, CONTENT_MODEL_VISUALDATA_JSONDATA );
+
+ $pageUpdater = $wikiPage->newPageUpdater( $this->user );
+ $pageUpdater->setContent( $targetSlot, $slotContent );
+ $comment = CommentStoreComment::newUnsavedComment( 'VisualData rename schema property' );
+ $revisionRecord = $pageUpdater->saveRevision( $comment, EDIT_INTERNAL );
+
+ if ( $revisionRecord === null ) {
+ $errors[] = 'saveRevision returned null';
+ return false;
+ }
+
+ return true;
+ }
+
+ /**
+ * Rebuild VisualData mirror tables without mutating the main RequestContext.
+ *
+ * @param RequestContext $context
+ * @param Title $title
+ * @param array $data
+ * @param array &$errors
+ * @return bool|null
+ */
+ private function rebuildArticleDataInternal( $context, $title, array $data, array &$errors ) {
+ if ( empty( $data['schemas'] ) ) {
+ return;
+ }
+
+ $schemas = array_keys( $data['schemas'] );
+ $schemas = \VisualData::getSchemas( $context, $schemas, true );
+
+ $databaseManager = new DatabaseManager();
+ $flatten = [];
+
+ foreach ( $data['schemas'] as $schemaName => $value ) {
+ if ( !array_key_exists( $schemaName, $schemas ) ) {
+ continue;
+ }
+
+ $flatten_ = $databaseManager->prepareData( $schemas[$schemaName], $value );
+ if ( $flatten_ === false ) {
+ $errors[] = 'error processing schema';
+ return false;
+ }
+
+ $flatten = array_merge( $flatten, $flatten_ );
+ }
+
+ $databaseManager->recordProperties( 'rename-schema-property', $title, $flatten, $errors );
+ return true;
+ }
+}
+
+$maintClass = RenameSchemaProperty::class;
+require_once RUN_MAINTENANCE_IF_MAIN;
\ No newline at end of file

Event Timeline

freephile created this paste.Jun 5 2026, 2:23 PM
freephile edited the content of this paste. (Show Details)Jun 5 2026, 2:37 PM

I apologize for not providing clear and atomic patches, but there is no project where I can track this.

The main thing I was hoping to share was the fix for removing mw-editsection artifacts that can arise https://phabricator.wikimedia.org/T427509

The whole new file is a utility maintenance script that can be used like an Admin could use "Special:ReplaceText" to rename a property in pages where you've renamed the schema property. I believe VisualData handles this through jobs like when you make an edit and the interface tells you that your change "will affect N pages", but I ran into some edge case where I had renamed a property in my schema and my data records (in JSON) or the database tables that mirror them for page queries fell out of sync. This was probably before I was aware of the DataRebuild script and the whole structure and workflow of the extension. In any case, the maintenance script might be a useful addition to VisualData.

Thomas-topway-it added a comment.Thu, Jun 18, 9:04 PM

@freephile, thanks a lot, merged as you know https://gerrit.wikimedia.org/r/c/mediawiki/extensions/VisualData/+/1302281/4

Regarding the delegation of schema-related operations to jobs, indeed the UpdateDataJob job was a work in progress that needs to be completed, so ideally both your maintenance script and that job could depend on a separate class encapsulating all the schema-related operations.

In this regarding, I wanted to ask: is the rename process recursive ? If not, perhaps a callback like in here could be used

Thomas-topway-it mentioned this in T427509: [VisualData] Strip or suppress mw:editsection at render/save time.Thu, Jun 18, 9:07 PM
Content licensed under Creative Commons Attribution-ShareAlike (CC BY-SA) 4.0 unless otherwise noted; code licensed under GNU General Public License (GPL) 2.0 or later and other open source licenses. By using this site, you agree to the Terms of Use, Privacy Policy, and Code of Conduct. · Wikimedia Foundation · Privacy Policy · Code of Conduct · Terms of Use · Disclaimer · CC-BY-SA · GPL · Credits