presto:wmf_content> select count(1) as count  from mediawiki_content_current_v1;
   count   
-----------
 663350101 
(1 row)

Query 20250424_135625_00522_53yev, FINISHED, 15 nodes
Splits: 8,790 total, 8,790 done (100.00%)
[Latency: client-side: 0:07, server-side: 0:07] [663M rows, 137MB] [99.8M rows/s, 20.6MB/s]



presto:wmf_content> select count(1) as count, wiki_id from mediawiki_content_current_v1 group by wiki_id order by count DESC;
   count   |       wiki_id        
-----------+----------------------
 153563110 | commonswiki          
 122225112 | wikidatawiki         
  62850374 | enwiki               
  19505632 | viwiki               
  13985201 | ruwikinews           
  13445925 | frwiki               
  12645582 | metawiki             
  11229772 | cebwiki              
  10089741 | enwiktionary         
   8725384 | arwiki               
   8392567 | eswiki      
....

Query 20250424_135816_00523_53yev, FINISHED, 15 nodes
Splits: 9,509 total, 9,509 done (100.00%)
[Latency: client-side: 0:10, server-side: 0:10] [663M rows, 137MB] [65.3M rows/s, 13.5MB/s]

presto:wmf_content> SELECT count(1) as count,
                 ->        wiki_id,
                 ->        page_id
                 -> FROM wmf_content.mediawiki_content_current_v1
                 -> GROUP BY wiki_id, page_id
                 -> HAVING count(1) > 1;
 count | wiki_id | page_id 
-------+---------+---------
(0 rows)

Query 20250424_140108_00525_53yev, FINISHED, 15 nodes
Splits: 9,269 total, 9,269 done (100.00%)
[Latency: client-side: 0:13, server-side: 0:13] [663M rows, 1.14GB] [50.1M rows/s, 87.9MB/s]

presto:wmf_content> SELECT wiki_id,
                 ->        page_id,
                 ->        revision_id
                 -> FROM wmf_content.mediawiki_content_current_v1
                 -> WHERE wiki_id IS NULL
                 ->    OR wiki_id = ''
                 ->    OR page_id IS NULL
                 ->    OR page_id < 0
                 ->    OR revision_id IS NULL
                 ->    OR revision_id < 0;
 wiki_id | page_id | revision_id 
---------+---------+-------------
(0 rows)

Query 20250424_140447_00527_53yev, FINISHED, 15 nodes
Splits: 8,789 total, 8,789 done (100.00%)
[Latency: client-side: 0:05, server-side: 0:05] [663M rows, 3.01GB] [143M rows/s, 664MB/s]

...
  File "/var/lib/hadoop/data/k/yarn/local/usercache/analytics/appcache/application_1741864027385_1386679/container_e133_1741864027385_1386679_01_000001/venv/lib/python3.10/site-packages/pyspark/python/lib/py4j-0.10.9.5-src.zip/py4j/protocol.py", line 330, in get_return_value
py4j.protocol.Py4JError: An error occurred while calling None.com.amazon.deequ.analyzers.Compliance. Trace:
py4j.Py4JException: Constructor com.amazon.deequ.analyzers.Compliance([class java.lang.String, class java.lang.String, class scala.None$]) does not exist
	at py4j.reflection.ReflectionEngine.getConstructor(ReflectionEngine.java:179)
	at py4j.reflection.ReflectionEngine.getConstructor(ReflectionEngine.java:196)
	at py4j.Gateway.invoke(Gateway.java:237)
	at py4j.commands.ConstructorCommand.invokeConstructor(ConstructorCommand.java:80)
	at py4j.commands.ConstructorCommand.execute(ConstructorCommand.java:69)
	at py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)
	at py4j.ClientServerConnection.run(ClientServerConnection.java:106)
	at java.lang.Thread.run(Thread.java:750)

<groupId>com.amazon.deequ</groupId>
<artifactId>deequ</artifactId>
<version>2.0.4-spark-3.1</version>

dependencies:
  - pip=23.1.2
  - python=3.10.11
  - pyspark=3.3.2
  - pip:
      - dnspython==2.5.0
...