Page MenuHomePhabricator

Selenium tests failing in Fresh since Mar 2021
Closed, ResolvedPublic

Description

I'm following instructions from Selenium/How-to/Run tests targeting MediaWiki-Docker using Fresh.

nobody@docker-desktop:/core$ npm run selenium-test

...

[0-1] AssertionError [ERR_ASSERTION] in "Page should be previewable"
Input A expected to strictly equal input B:
+ expected - actual
- 'beforeEach-content-0.0050359081519923965-Iñtërnâtiônàlizætiøn\n\n'
+ 'beforeEach-content-0.0050359081519923965-Iñtërnâtiônàlizætiøn'

[0-1] Error in "Page should be creatable"
Timeout of 60000ms exceeded. For async tests and hooks, ensure "done()" is called; if returning a Promise, ensure it resolves. (/core/tests/selenium/specs/page.js)
[0-1] Error in "Page should be re-creatable"
Timeout of 60000ms exceeded. For async tests and hooks, ensure "done()" is called; if returning a Promise, ensure it resolves. (/core/tests/selenium/specs/page.js)
[0-1] Error in "Page should be editable @daily"
Timeout of 60000ms exceeded. For async tests and hooks, ensure "done()" is called; if returning a Promise, ensure it resolves. (/core/tests/selenium/specs/page.js)
[0-1] Error in "Page should have history @daily"
Timeout of 60000ms exceeded. For async tests and hooks, ensure "done()" is called; if returning a Promise, ensure it resolves. (/core/tests/selenium/specs/page.js)
[0-1] Error in "Page should be deletable"
Timeout of 60000ms exceeded. For async tests and hooks, ensure "done()" is called; if returning a Promise, ensure it resolves. (/core/tests/selenium/specs/page.js)
[0-1] Error in "Page should be restorable"
Timeout of 60000ms exceeded. For async tests and hooks, ensure "done()" is called; if returning a Promise, ensure it resolves. (/core/tests/selenium/specs/page.js)
[0-1] Error in "Page should be undoable"
Timeout of 60000ms exceeded. For async tests and hooks, ensure "done()" is called; if returning a Promise, ensure it resolves. (/core/tests/selenium/specs/page.js)

...

Spec Files:      5 passed, 1 failed, 6 total (100% completed) in 00:08:49

Full output is at P14745.

Event Timeline

A simple change fixes timeout errors and speeds up the tests. See P14746.

nobody@docker-desktop:/core$ git diff
diff --git a/tests/selenium/specs/page.js b/tests/selenium/specs/page.js
index 74408ccae1..042ec81279 100644
--- a/tests/selenium/specs/page.js
+++ b/tests/selenium/specs/page.js
@@ -18,7 +18,7 @@ describe( 'Page', function () {
        } );
 
        beforeEach( function () {
-               browser.deleteAllCookies();
+               browser.reloadSession();
                content = Util.getTestString( 'beforeEach-content-' );
                name = Util.getTestString( 'BeforeEach-name-' );
        } );
nobody@docker-desktop:/core$ npm run selenium-test
...
Spec Files:      5 passed, 1 failed, 6 total (100% completed) in 00:02:38

I've added a few newlines to a few assertions and it fixed all errors. See P14747.

nobody@docker-desktop:/core$ git diff
diff --git a/tests/selenium/specs/page.js b/tests/selenium/specs/page.js
index 74408ccae1..4bdfab5e14 100644
--- a/tests/selenium/specs/page.js
+++ b/tests/selenium/specs/page.js
@@ -18,7 +18,7 @@ describe( 'Page', function () {
        } );
 
        beforeEach( function () {
-               browser.deleteAllCookies();
+               browser.reloadSession();
                content = Util.getTestString( 'beforeEach-content-' );
                name = Util.getTestString( 'BeforeEach-name-' );
        } );
@@ -27,7 +27,7 @@ describe( 'Page', function () {
                EditPage.preview( name, content );
 
                assert.strictEqual( EditPage.heading.getText(), 'Creating ' + name );
-               assert.strictEqual( EditPage.displayedContent.getText(), content );
+               assert.strictEqual( EditPage.displayedContent.getText(), `${content}\n\n` );
                assert( EditPage.content.isDisplayed(), 'editor is still present' );
                assert( !EditPage.conflictingContent.isDisplayed(), 'no edit conflict happened' );
 
@@ -43,7 +43,7 @@ describe( 'Page', function () {
 
                // check
                assert.strictEqual( EditPage.heading.getText(), name );
-               assert.strictEqual( EditPage.displayedContent.getText(), content );
+               assert.strictEqual( EditPage.displayedContent.getText(), `${content}\n\n` );
        } );
 
        it( 'should be re-creatable', function () {
@@ -60,7 +60,7 @@ describe( 'Page', function () {
 
                // check
                assert.strictEqual( EditPage.heading.getText(), name );
-               assert.strictEqual( EditPage.displayedContent.getText(), content );
+               assert.strictEqual( EditPage.displayedContent.getText(), `${content}\n\n` );
        } );
 
        it( 'should be editable @daily', function () {
@@ -104,7 +104,7 @@ describe( 'Page', function () {
                // check
                assert.strictEqual(
                        DeletePage.displayedContent.getText(),
-                       '"' + name + '" has been deleted. See deletion log for a record of recent deletions.\n\nReturn to Main Page.'
+                       '"' + name + '" has been deleted. See deletion log for a record of recent deletions.\n\nReturn to Main Page.\n\n'
                );
        } );
 
@@ -122,7 +122,7 @@ describe( 'Page', function () {
                RestorePage.restore( name, 'restore reason' );
 
                // check
-               assert.strictEqual( RestorePage.displayedContent.getText(), name + ' has been restored\n\nConsult the deletion log for a record of recent deletions and re
storations.' );
+               assert.strictEqual( RestorePage.displayedContent.getText(), name + ' has been restored\n\nConsult the deletion log for a record of recent deletions an
d restorations.\n\n' );
        } );
 
        it( 'should be undoable', function () {
@@ -139,7 +139,7 @@ describe( 'Page', function () {
 
                UndoPage.undo( name, previousRev, undoRev );
 
-               assert.strictEqual( EditPage.displayedContent.getText(), content );
+               assert.strictEqual( EditPage.displayedContent.getText(), `${content}\n\n` );
        } );
 
 } );
nobody@docker-desktop:/core$ npm run selenium-test
...
Spec Files:      6 passed, 6 total (100% completed) in 00:02:38

Change 670492 had a related patch set uploaded (by Zfilipin; owner: Zfilipin):
[mediawiki/core@master] WIP selenium: Fix test when running in Fresh

https://gerrit.wikimedia.org/r/670492

The problem is that the above patch fixes the tests for Fresh, but fails in CI.

Change 672172 had a related patch set uploaded (by Zfilipin; owner: Jared Blumer):
[mediawiki/core@master] selenium: Update documentation in DEVELOPERS.md

https://gerrit.wikimedia.org/r/672172

Change 670492 abandoned by Zfilipin:
[mediawiki/core@master] WIP selenium: Fix test when running in Fresh

Reason:
I don't know why tests fail in Fresh.

https://gerrit.wikimedia.org/r/670492

@Krinkle do you know why tests fail when targeting Fresh?

They've generally not failed for me. It passed when we set up Wdio in core, it passed when we upgraded to Wdio 5 and later 6. And it passed a month or two ago when we debugged it together. I'll try to give it another go this or next week to see if it's failing now.

@Krinkle yes, tests used to pass. I've noticed they started failing recently. It might be connected to 666946.

Change 672172 merged by jenkins-bot:

[mediawiki/core@master] selenium: Update documentation in DEVELOPERS.md

https://gerrit.wikimedia.org/r/672172

@Krinkle yes, tests used to pass. I've noticed they started failing recently. It might be connected to 666946.

Thanks, that makes sense. I can reproduce it as well. This change exposed the underlying chromedriver from CI, and that in turn was upgraded from Chrome 71 and Chrome 73 around the same time (T216702). This updated dockerimage has been released in WMF CI for the quibble-selenium and node-test jobs, but Fresh has not been updated yet.

The following change makes it pass:

diff
--- a/bin/fresh-node10
+++ b/bin/fresh-node10
@@ -4,9 +4,9 @@
 imagename=docker-registry.wikimedia.org/releng/node10-test-browser
-imageversion=0.6.2
+imageversion=0.6.3-s2
# fresh: 21.04.1-dev (2021-04-29)
# image: docker-registry.wikimedia.org/releng/node10-test-browser:0.6.3-s2
#           Chromium 73.0.3683.75
# […]

nobody@docker-desktop:/mediawiki$ npm run selenium-test

> @ selenium-test /mediawiki
> wdio ./tests/selenium/wdio.conf.js


Execution of 5 spec files started at 2021-04-30T00:04:11.921Z

(node:20) ExperimentalWarning: The dns.promises API is experimental
(node:20) ExperimentalWarning: The fs.promises API is experimental
[0-0] (node:29) ExperimentalWarning: The dns.promises API is experimental
[0-0] (node:29) ExperimentalWarning: The fs.promises API is experimental
[0-0] RUNNING in chrome - /tests/selenium/wdio-mediawiki/specs/BlankPage.js
[0-0] PASSED in chrome - /tests/selenium/wdio-mediawiki/specs/BlankPage.js
[0-1] (node:131) ExperimentalWarning: The dns.promises API is experimental
[0-1] (node:131) ExperimentalWarning: The fs.promises API is experimental
[0-1] RUNNING in chrome - /tests/selenium/specs/page.js
[0-1] [00:04:38] [S] [MWBOT] Login successful: Admin@http://default.web.mw.localhost:8080/mediawiki
[0-1] PASSED in chrome - /tests/selenium/specs/page.js
[0-2] (node:432) ExperimentalWarning: The dns.promises API is experimental
[0-2] (node:432) ExperimentalWarning: The fs.promises API is experimental
[0-2] RUNNING in chrome - /tests/selenium/specs/specialrecentchanges.js
[0-2] [00:06:34] [S] [MWBOT] Login successful: Admin@http://default.web.mw.localhost:8080/mediawiki
[0-2] PASSED in chrome - /tests/selenium/specs/specialrecentchanges.js
[0-3] (node:534) ExperimentalWarning: The dns.promises API is experimental
[0-3] (node:534) ExperimentalWarning: The fs.promises API is experimental
[0-3] RUNNING in chrome - /tests/selenium/specs/specialwatchlist.js
[0-3] [00:07:04] [S] [MWBOT] Login successful: Admin@http://default.web.mw.localhost:8080/mediawiki
[0-3] PASSED in chrome - /tests/selenium/specs/specialwatchlist.js
[0-4] (node:639) ExperimentalWarning: The dns.promises API is experimental
[0-4] (node:639) ExperimentalWarning: The fs.promises API is experimental
[0-4] RUNNING in chrome - /tests/selenium/specs/user.js
[0-4] [00:07:53] [S] [MWBOT] Login successful: Admin@http://default.web.mw.localhost:8080/mediawiki
[0-4] PASSED in chrome - /tests/selenium/specs/user.js

 "dot" Reporter:
.............

Spec Files:	 5 passed, 5 total (100% completed) in 00:04:12

I'll make a release for this.

Change 683756 had a related patch set uploaded (by Krinkle; author: Krinkle):

[fresh@master] Tag 21.04.1

https://gerrit.wikimedia.org/r/683756

Change 683756 merged by jenkins-bot:

[fresh@master] Tag 21.04.1

https://gerrit.wikimedia.org/r/683756

Krinkle renamed this task from Selenium tests fail when using Fresh to Selenium tests failing in Fresh since Mar 2021.May 10 2021, 1:29 AM
Krinkle moved this task from Inbox to Problem on the Fresh board.