rMGRA/grabFiles.php
b49e5e00264fmaster

grabFiles.php
View Options

			<?php
			/**
			* Grabs files from a pre-existing wiki into a new wiki.
			* Merge back into grabImages or something later.
			*
			* @file
			* @ingroup Maintenance
			* @author Calimonious the Estrange
			* @author Jesús Martínez <martineznovo@gmail.com>
			* @date 31 December 2012
			* @version 1.0
			* @note Based on code by Misza, Jack Phoenix and Edward Chernenko.
			*/

			require_once 'includes/FileGrabber.php';

			class GrabFiles extends FileGrabber {

			protected ?string $endDate;

			public function __construct() {
			parent::__construct();
			$this->addDescription(
			'Grabs files from a pre-existing wiki into a new wiki, using file upload configuration of the local wiki.'
			);
			$this->addOption( 'from', 'Name of file to start from', false, true );
			$this->addOption( 'enddate', 'Date after which to ignore new files (20121222142317, 2012-12-22T14:23:17Z, etc)', false, true );
			}

			public function execute() {
			parent::execute();

			$this->endDate = $this->getOption( 'enddate' );
			if ( $this->endDate ) {
			$this->endDate = wfTimestamp( TS_MW, $this->endDate );
			if ( !$this->endDate ) {
			$this->fatalError( 'Invalid enddate format.' );
			}
			} else {
			$this->endDate = wfTimestampNow();
			}

			$params = [
			'generator' => 'allimages',
			'gailimit' => 'max',
			'prop' => 'imageinfo',
			'iiprop' => 'timestamp\|user\|userid\|comment\|url\|size\|sha1\|mime\|metadata\|archivename\|bitdepth\|mediatype',
			'iilimit' => 'max'
			];

			$gaifrom = $this->getOption( 'from' );
			$more = true;
			$count = 0;

			if ( $gaifrom !== null ) {
			$params['gaifrom'] = $gaifrom;
			}

			$this->output( "Processing and downloading files...\n" );
			while ( $more ) {
			$result = $this->bot->query( $params );
			if ( empty( $result['query']['pages'] ) ) {
			$this->fatalError( 'No files found...' );
			}

			foreach ( $result['query']['pages'] as $file ) {
			$count = $count + $this->processFile( $file );
			}

			if ( isset( $result['query-continue'] ) ) {
			$gaifrom = $result['query-continue']['allimages']['gaifrom'];
			} elseif ( isset( $result['continue'] ) ) {
			$params = array_merge( $params, $result['continue'] );
			} else {
			$more = false;
			}
			}
			$this->output( "$count files downloaded.\n" );
			}

			/**
			* Process the information from a given file returned by the api
			*
			* @param array $entry Page data returned from the api with imageinfo
			* @return int Number of image revisions processed.
			*/
			function processFile( $entry ) {
			$name = $this->sanitiseTitle( $entry['ns'], $entry['title'] );

			# Check if file already exists.
			# NOTE: wfFindFile() checks foreign repos too. Use local repo only
			# newFile skips supression checks
			$file = $this->localRepo->newFile( $name );
			if ( $file->exists() ) {
			return 0;
			}

			$this->output( "Processing {$name}: " );
			$count = 0;

			foreach ( $entry['imageinfo'] as $fileVersion ) {
			# Api returns file revisions from new to old.
			# WARNING: If a new version of a file is uploaded after the start of the script
			# (or endDate), the file and all its previous revisions would be skipped,
			# potentially leaving pages that were using the old image with redlinks.
			# To prevent this, we'll skip only more recent versions, and mark the first
			# one before the end date as the latest
			if ( !$count && wfTimestamp( TS_MW, $fileVersion['timestamp'] ) > $this->endDate ) {
			#return 0;
			continue;
			}

			# Check for Wikia's videos
			if ( $this->isWikiaVideo( $fileVersion ) ) {
			$this->output( "...this appears to be a video, skipping it.\n" );
			return 0;
			}

			if ( $count > 0 ) {
			$status = $this->oldUpload( $name, $fileVersion );
			} else {
			$status = $this->newUpload( $name, $fileVersion );
			}

			if ( $status->isOK() ) {
			$count++;
			}
			}
			if ( $count == 1 ) {
			$this->output( "1 revision\n" );
			} else {
			$this->output( "$count revisions\n" );
			}

			return $count;
			}
			}

			$maintClass = 'GrabFiles';
			require_once RUN_MAINTENANCE_IF_MAIN;

rMGRA/grabFiles.phpb49e5e00264fmaster

grabFiles.phpView Options

rMGRA/grabFiles.php
b49e5e00264fmaster

grabFiles.php
View Options