Replag via heartbeat

slice => lag */ $replag = array(); /** @var array $wikis dbname => slice */ $wikis = array(); // Read database credentials from replica.my.cnf $cnf = parse_ini_file( posix_getpwuid( posix_getuid() )['dir'] . '/replica.my.cnf' ); /** * Connect to a MySQL database. * @param string $db Database name * @param string $host Database server * @return PDO Database connection */ function connect( $db, $host ) { global $cnf; return new PDO( "mysql:dbname={$db};host={$host}", $cnf['user'], $cnf['password'], array( PDO::ATTR_TIMEOUT => 5, PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, ) ); } /** * Format a count of seconds as a pretty time interval. * @param int $seconds * @return string Time interval in :: format */ function secondsAsTime( $seconds ) { + if ( $seconds === PHP_INT_MAX ) { + return 'N/A'; + } return sprintf( '%02d:%02d:%02d', floor( $seconds / 3600 ), ( $seconds / 60 ) % 60, $seconds % 60 ); } // Get lag data for each slice from the heartbeat_p db on each host foreach ( $cluster as $host) { $replag[$host] = array(); foreach ( $slices as $slice ) { - $dbh = connect( 'heartbeat_p', $host ); - $stmt = $dbh->prepare( - 'SELECT lag FROM heartbeat WHERE shard = ?' ); - $stmt->execute( array( $slice ) ); - $replag[$host][$slice] = $stmt->fetchColumn(); - $stmt->closeCursor(); + try { + $dbh = connect( 'heartbeat_p', $host ); + $stmt = $dbh->prepare( + 'SELECT lag FROM heartbeat WHERE shard = ?' ); + $stmt->execute( array( $slice ) ); + $replag[$host][$slice] = $stmt->fetchColumn(); + $stmt->closeCursor(); + } catch ( PDOException $e ) { + $replag[$host][$slice] = PHP_INT_MAX; + } } } // Print replag data for each shard on each host foreach ( $replag as $host => $shards ) { $shost = htmlspecialchars( $host ); ?> $lag ) { echo ''; echo ''; echo ''; echo ''; } ?>


Shard	Lag (seconds)	Lag (time)
', htmlspecialchars( $shard ), '	', htmlspecialchars( $lag ), '	', secondsAsTime( $lag ), '

query( 'SELECT dbname, slice FROM wiki ORDER BY dbname' ); -$res = $stmt->fetchAll( PDO::FETCH_ASSOC ); -$stmt->closeCursor(); +try { + // Get list of all databases and the slices they live on from meta_p.wiki + $dbh = connect( 'meta_p', 's7.web.db.svc.eqiad.wmflabs' ); + $stmt = $dbh->query( 'SELECT dbname, slice FROM wiki ORDER BY dbname' ); + $res = $stmt->fetchAll( PDO::FETCH_ASSOC ); + $stmt->closeCursor(); -// Populate $wikis and $slices from meta_p.wiki results -foreach ( $res as $row ) { - list( $slice, $domain ) = explode( '.', $row['slice'] ); - $wikis[$row['dbname']] = $slice; - $slices[$slice] = $row['slice']; + // Populate $wikis and $slices from meta_p.wiki results + foreach ( $res as $row ) { + list( $slice, $domain ) = explode( '.', $row['slice'] ); + $wikis[$row['dbname']] = $slice; + $slices[$slice] = $row['slice']; + } +} catch ( PDOException $e ) { + // TODO: better error reporting } + // Get lag data for each slice from the heartbeat_p db on the matching host foreach ( $slices as $slice => $host ) { - $dbh = connect( 'heartbeat_p', $host ); - $stmt = $dbh->prepare( 'SELECT lag FROM heartbeat WHERE shard = ?' ); - $stmt->execute( array( $slice ) ); - $replag[$slice] = $stmt->fetchColumn(); - $stmt->closeCursor(); + try { + $dbh = connect( 'heartbeat_p', $host ); + $stmt = $dbh->prepare( 'SELECT lag FROM heartbeat WHERE shard = ?' ); + $stmt->execute( array( $slice ) ); + $replag[$slice] = $stmt->fetchColumn(); + $stmt->closeCursor(); + } catch ( PDOException $e ) { + $replag[$slice] = PHP_INT_MAX; + } } ?> $shard ) { $lag = $replag[$shard]; echo ''; echo ''; echo ''; echo ''; echo ''; } ?>

Database	Shard	Lag (seconds)	Lag (time)
', htmlspecialchars( $wiki ), '.labsdb	', htmlspecialchars( $shard ), '	', htmlspecialchars( $lag ), '	', secondsAsTime( $lag ), '