Allow patches to be updated

deduplicate.sh

@@ -12,4 +12,7 @@ BASEDIR=$(dirname "$0")
 # I'd have to figure out how to provision instances using XFS or something.
 # (Also, `rdfind` doesn't have an option for that, `rmlint` could be used.)
 
-sudo -u www-data rdfind -makehardlinks true -makeresultsfile false -checksum sha1 $BASEDIR/wikis
+# Only loook inside the "w" folder as while we are updating a wiki it is
+# called "w-updating" and shouldn't be deduplicated.
+find $BASEDIR/wikis -name "w" -type d \
+	xargs sudo -u www-data rdfind -makehardlinks true -makeresultsfile false -checksum sha1

delete.php

@@ -28,7 +28,7 @@
 			'<td data-label="Wiki" class="wiki"><a href="wikis/' . $wiki . '/w" title="' . $wiki . '">' . substr( $wiki, 0, 10 ) . '</a></td>' .
 			'<td data-label="Patches" class="patches">' . $patches . '</td>' .
 			'<td data-label="Linked tasks" class="linkedTasks">' . $linkedTasks . '</td>' .
-			'<td data-label="Time" class="date">' . date( 'Y-m-d H:i:s', $wikiData[ 'created' ] ) . '</td>' .
+			'<td data-label="Time" class="date">' . date( 'Y-m-d H:i:s', $wikiData[ 'updated' ] ) . '</td>' .
 			( $useOAuth ? '<td data-label="Creator">' . ( $creator ? user_link( $creator ) : '?' ) . '</td>' : '' ) .
 		'</tr>' .
 	'</table>';

editcounts.php

@@ -47,7 +47,7 @@
 	],
 ];
 
-$results = $mysqli->query( 'SELECT wiki FROM wikis WHERE !deleted ORDER BY created DESC' );
+$results = $mysqli->query( 'SELECT wiki FROM wikis WHERE !deleted ORDER BY updated DESC' );
 if ( !$results ) {
 	die( $mysqli->error );
 }

includes.php

@@ -36,13 +36,21 @@ function insert_wiki_data( string $wiki, string $creator, int $created, string $
 	global $mysqli;
 	$stmt = $mysqli->prepare( '
 		INSERT INTO wikis
-		(wiki, creator, created, branch)
-		VALUES(?, ?, FROM_UNIXTIME(?), ?)
+		(wiki, creator, created, updated, branch)
+		VALUES(?, ?, FROM_UNIXTIME(?), FROM_UNIXTIME(?), ?)
 	' );
 	if ( !$stmt ) {
 		echo $mysqli->error;
 	}
-	$stmt->bind_param( 'ssis', $wiki, $creator, $created, $branch );
+	$stmt->bind_param( 'ssiis', $wiki, $creator, $created, $created, $branch );
+	$stmt->execute();
+	$stmt->close();
+}
+
+function wiki_update_timestamp( string $wiki ) {
+	global $mysqli;
+	$stmt = $mysqli->prepare( 'UPDATE wikis SET updated = NOW() WHERE wiki = ?' );
+	$stmt->bind_param( 's', $wiki );
 	$stmt->execute();
 	$stmt->close();
 }
@@ -77,7 +85,7 @@ function get_wiki_data( string $wiki ): array {
 	global $mysqli;
 
 	$stmt = $mysqli->prepare( '
-		SELECT wiki, creator, UNIX_TIMESTAMP( created ) created, patches, branch, announcedTasks, timeToCreate, deleted
+		SELECT wiki, creator, UNIX_TIMESTAMP( created ) created, UNIX_TIMESTAMP( updated ) updated, patches, branch, announcedTasks, timeToCreate, deleted
 		FROM wikis WHERE wiki = ?
 	' );
 	if ( !$stmt ) {

index.php

@@ -281,10 +281,10 @@
 $username = $user ? $user->username : null;
 
 $stmt = $mysqli->prepare( '
-	SELECT wiki, creator, UNIX_TIMESTAMP( created ) created, patches, branch, announcedTasks, timeToCreate, deleted
+	SELECT wiki, creator, UNIX_TIMESTAMP( updated ) updated, patches, branch, announcedTasks, timeToCreate, deleted
 	FROM wikis
 	WHERE !deleted
-	ORDER BY IF( creator = ?, 1, 0 ) DESC, created DESC
+	ORDER BY IF( creator = ?, 1, 0 ) DESC, updated DESC
 ' );
 if ( !$stmt ) {
 	die( $mysqli->error );
@@ -330,6 +330,7 @@
 
 	$actions = [];
 	if ( $canDelete ) {
+		$actions[] = '<a href="update.php?wiki=' . $wiki . '">Update</a>';
 		$actions[] = '<a href="delete.php?wiki=' . $wiki . '">Delete</a>';
 	}
 	if ( $canCreate ) {
@@ -346,7 +347,7 @@
 		'</td>' .
 		'<td data-label="Patches" class="patches">' . $patches . '</td>' .
 		'<td data-label="Linked tasks" class="linkedTasks">' . $linkedTasks . '</td>' .
-		'<td data-label="Time" class="date">' . date( 'Y-m-d H:i:s', $wikiData[ 'created' ] ) . '</td>' .
+		'<td data-label="Time" class="date">' . date( 'Y-m-d H:i:s', $wikiData[ 'updated' ] ) . '</td>' .
 		( $useOAuth ? '<td data-label="Creator">' . ( $creator ? user_link( $creator ) : '?' ) . '</td>' : '' ) .
 		( $canAdmin ? '<td data-label="Time to create">' . ( $wikiData['timeToCreate'] ? format_duration( $wikiData['timeToCreate'] ) : '' ) . '</td>' : '' ) .
 		( count( $actions ) ?

new/applypatch.sh

@@ -3,6 +3,9 @@ set -ex
 
 cd $PATCHDEMO/wikis/$NAME/$REPO
 
+# Required when updating an existing wiki
+git reset --hard origin/master
+
 git fetch origin $REF
 
 # Apply $HASH and its parent commits up to $BASE on top of current HEAD.

new/postupdate.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+set -ex
+
+# run update script (#166, #244)
+php $PATCHDEMO/wikis/$NAME/w/maintenance/update.php --quick
+
+# Update Main_Page
+MAINPAGETITLE=$( echo 'echo Title::newMainPage()->getPrefixedText();' | php $PATCHDEMO/wikis/$NAME/w/maintenance/eval.php 2> /dev/null )
+MAINPAGECURRENT=$( php $PATCHDEMO/wikis/$NAME/w/maintenance/getText.php "$MAINPAGETITLE" )
+echo "$MAINPAGECURRENT$MAINPAGE" | php $PATCHDEMO/wikis/$NAME/w/maintenance/edit.php "$MAINPAGETITLE" || echo "Can't edit main page"

sql/patchdemo.sql

@@ -42,3 +42,9 @@ ALTER TABLE `tasks`
 
 ALTER TABLE `wikis`
 	ADD COLUMN IF NOT EXISTS `branch` VARCHAR(64) NOT NULL AFTER `patches`;
+
+ALTER TABLE `wikis`
+	ADD COLUMN `updated` DATETIME NOT NULL AFTER `created`,
+	ADD INDEX `updated` (`updated`);
+
+UPDATE wikis SET updated = created;

update.php

@@ -0,0 +1,205 @@
+<?php
+
+use Symfony\Component\Yaml\Yaml;
+
+require_once "includes.php";
+
+include "header.php";
+
+ob_implicit_flush( true );
+
+if ( $useOAuth && !$user ) {
+	echo oauth_signin_prompt();
+	die();
+}
+
+$wiki = $_GET['wiki'];
+$wikiData = get_wiki_data( $wiki );
+
+if ( !can_delete( $wikiData['creator'] ) ) {
+	die( '<p>You are not allowed to update this wiki.</p>' );
+}
+
+function abandon( string $errHtml ) {
+	die( $errHtml );
+}
+
+echo '<p>Updating wiki <a class="wiki" href="wikis/' . $wiki . '/w" title="' . $wiki . '">' . $wiki . '</a>.</p>';
+
+echo '<div class="consoleLog">';
+
+ob_flush();
+
+$baseEnv = [
+	'PATCHDEMO' => __DIR__,
+	'NAME' => $wiki,
+];
+
+$patchesApplied = [];
+$patchesToUpdate = [];
+$linkedTasks = [];
+$commands = [];
+$usedRepos = [];
+
+foreach ( $wikiData['patchList'] as $patch => $patchData ) {
+	$r = $patchData['r'];
+	$data = gerrit_query( "changes/?q=change:$r&o=LABELS&o=CURRENT_REVISION", true );
+
+	// get the info
+	$repo = $data[0]['project'];
+	$base = 'origin/' . $data[0]['branch'];
+	$revision = $data[0]['current_revision'];
+	$ref = $data[0]['revisions'][$revision]['ref'];
+	$id = $data[0]['id'];
+
+	$repos = get_repo_data( 'w-updating/' );
+	if ( !isset( $repos[ $repo ] ) ) {
+		$repo = htmlentities( $repo );
+		abandon( "Repository <em>$repo</em> not supported" );
+	}
+	$path = $repos[ $repo ];
+	$usedRepos[] = $repo;
+
+	if (
+		$config[ 'requireVerified' ] &&
+		( $data[0]['labels']['Verified']['approved']['_account_id'] ?? null ) !== 75
+	) {
+		// The patch doesn't have V+2, check if the uploader is trusted
+		$uploaderId = $data[0]['revisions'][$revision]['uploader']['_account_id'];
+		$uploader = gerrit_query( 'accounts/' . $uploaderId, true );
+		if ( !is_trusted_user( $uploader['email'] ) ) {
+			abandon( "Patch must be approved (Verified+2) by jenkins-bot, or uploaded by a trusted user" );
+		}
+	}
+
+	$patch = $data[0]['_number'] . ',' . $data[0]['revisions'][$revision]['_number'];
+	$patchesApplied[] = $patch;
+
+	$r = $patchData['r'];
+	$pOld = (int)$patchData['p'];
+	$pNew = $data[0]['revisions'][$revision]['_number'];
+	if ( $pNew > $pOld ) {
+		echo "<strong>Updating change $r from patchset $pOld to $pNew.</strong>";
+	} else {
+		echo "<strong>Change $r is already using the latest patchset ($pOld).</strong>";
+		continue;
+	}
+
+	$patchesToUpdate[] = $patch;
+
+	$commands[] = [
+		[
+			'REPO' => $path,
+			'REF' => $ref,
+			'BASE' => $base,
+			'HASH' => $revision,
+		],
+		__DIR__ . '/new/applypatch.sh'
+	];
+
+	$relatedChanges = [];
+	$relatedChanges[] = [ $data[0]['_number'], $data[0]['revisions'][$revision]['_number'] ];
+
+	// Look at all commits in this patch's tree for cross-repo dependencies to add
+	$data = gerrit_query( "changes/$id/revisions/$revision/related", true );
+	// Ancestor commits only, not descendants
+	$foundCurr = false;
+	foreach ( $data['changes'] as $change ) {
+		if ( $foundCurr ) {
+			// Querying by change number is allegedly deprecated, but the /related API doesn't return the 'id'
+			$relatedChanges[] = [ $change['_change_number'], $change['_revision_number'] ];
+		}
+		$foundCurr = $foundCurr || $change['commit']['commit'] === $revision;
+	}
+
+	foreach ( $relatedChanges as [ $c, $r ] ) {
+		$data = gerrit_query( "changes/$c/revisions/$r/commit", true );
+
+		preg_match_all( '/^Depends-On: (.+)$/m', $data['message'], $m );
+		foreach ( $m[1] as $changeid ) {
+			if ( !in_array( $changeid, $patches, true ) ) {
+				// The entry we add here will be processed by the topmost foreach
+				$patches[] = $changeid;
+			}
+		}
+	}
+	ob_flush();
+}
+$usedRepos = array_unique( $usedRepos );
+
+if ( !count( $commands ) ) {
+	abandon( 'No patches to update.' );
+}
+
+$error = shell_echo( __DIR__ . '/new/unlinkbefore.sh', $baseEnv );
+if ( $error ) {
+	abandon( "Could not copy wiki files to update." );
+}
+
+// The working directory is now w-updating/, which is ignored by the rdfind cron job (deduplicate.sh)
+// This means $wgScriptPath is incorrect, so don't try to run any MW scripts.
+
+foreach ( $commands as $i => $command ) {
+	$error = shell_echo( $command[1], $baseEnv + $command[0] );
+	if ( $error ) {
+		abandon( "Could not apply patch {$patchesToUpdate[$i]}" );
+	}
+	ob_flush();
+}
+
+$composerInstallRepos = Yaml::parse( file_get_contents( __DIR__ . '/repository-lists/composerinstall.yaml' ) );
+foreach ( $usedRepos as $repo ) {
+	if ( in_array( $repo, $composerInstallRepos, true ) ) {
+		$error = shell_echo( __DIR__ . '/new/composerinstall.sh',
+			$baseEnv + [
+				// Variable used by composer itself, not our script
+				'COMPOSER_HOME' => __DIR__ . '/composer',
+				'REPO_TARGET' => $repos[$repo],
+			]
+		);
+		if ( $error ) {
+			abandon( "Could not fetch dependencies for <em>$repo</em>" );
+		}
+		ob_flush();
+	}
+}
+
+$error = shell_echo( __DIR__ . '/new/unlinkafter.sh', $baseEnv );
+if ( $error ) {
+	abandon( "Could not overwrite wiki files after updating." );
+}
+
+// The working directory is back to w/. Scripts beyond this point should not modify the filesystem.
+
+$mainPage = "\n\nThis wiki was updated on ~~~~~ with the following newer patches:";
+foreach ( $patchesToUpdate as $patch ) {
+	preg_match( '`([0-9]+),([0-9]+)`', $patch, $matches );
+	list( $t, $r, $p ) = $matches;
+
+	$data = gerrit_query( "changes/$r/revisions/$p/commit", true );
+	if ( $data ) {
+		$t = $t . ': ' . $data[ 'subject' ];
+		get_linked_tasks( $data['message'], $linkedTasks );
+	}
+
+	$t = htmlentities( $t );
+
+	$mainPage .= "\n:* [{$config['gerritUrl']}/r/c/$r/$p <nowiki>$t</nowiki>]";
+}
+
+$error = shell_echo( __DIR__ . '/new/postupdate.sh',
+	$baseEnv + [
+		'MAINPAGE' => $mainPage,
+	]
+);
+if ( $error ) {
+	abandon( "Could not update wiki content" );
+}
+
+// Update DB record with _all_ patches applied (include those which weren't updated)
+wiki_add_patches( $wiki, $patchesApplied );
+wiki_update_timestamp( $wiki );
+
+echo "Done!";
+
+echo '</div>';