ssh an-coord1001.eqiad.wmnet
sudo -u analytics bash

kerberos-run-command analytics spark3-sql

spark-sql (default)> CREATE TABLE wmf_dumps.wikitext_raw_rc1 (
                   >     page_id                     BIGINT    COMMENT 'id of the page',
                   >     page_namespace              INT       COMMENT 'namespace of the page',
                   >     page_title                  STRING    COMMENT 'title of the page ',
                   >     page_redirect_title         STRING    COMMENT 'title of the redirected-to page', 
                   >     user_id                     BIGINT    COMMENT 'id of the user that made the revision (or -1 if anonymous)',
                   >     user_text                   STRING    COMMENT 'text of the user that made the revision (either username or IP)',
                   >     user_is_visible             BOOLEAN   COMMENT 'whether the user details have been suppressed or not',
                   >     revision_id                 BIGINT    COMMENT 'id of the revision',
                   >     revision_parent_id          BIGINT    COMMENT 'id of the parent revision',
                   >     revision_timestamp          TIMESTAMP COMMENT 'timestamp of the revision',
                   >     revision_is_minor_edit      BOOLEAN   COMMENT 'whether this revision is a minor edit or not',
                   >     revision_comment            STRING    COMMENT 'comment made with revision',
                   >     revision_comment_is_visible BOOLEAN   COMMENT 'whether the comment has been suppressed or not',
                   >     revision_sha1               STRING    COMMENT 'Nested SHA1 hash of hashes of all content slots. See https://www.mediawiki.org/wiki/Manual:Revision_table#rev_sha1',
                   >     revision_size               BIGINT    COMMENT 'the sum of the content_size of all content slots',
                   >     revision_content_slots      MAP<
                   >                                     STRING,
                   >                                     STRUCT<content_body:   STRING,
                   >                                            content_format: STRING,
                   >                                            content_model:  STRING,
                   >                                            content_sha1:   STRING,
                   >                                            content_size:   BIGINT
                   >                                     >
                   >                                 >         COMMENT 'a MAP containing all the content slots associated to this revision. Typically just the "main" slot, but also "mediainfo" for commonswiki.',
                   >     revision_content_is_visible BOOLEAN   COMMENT 'whether the content in the slots has been suppressed or not',
                   >     wiki_db                     STRING    COMMENT 'the wiki project',
                   >     row_last_update             TIMESTAMP COMMENT 'the timestamp of the last event or backfill that updated this row'
                   > )
                   > USING ICEBERG
                   > PARTITIONED BY (wiki_db, days(revision_timestamp)) -- wiki_db partitioning helps the backfill process,
                   >                                                    -- while days(revision_timestamp) helps the hourly stream intake
                   > TBLPROPERTIES (
                   >     'write.format.default' = 'avro',               -- avoid OOMs due to revision_content_slots
                   >     'write.metadata.previous-versions-max' = '10',
                   >     'write.metadata.delete-after-commit.enabled' = 'true'
                   > )
                   > COMMENT 'Base to create MediaWiki full revision dumps from.';
Response code
Time taken: 23.802 seconds

kerberos-run-command analytics spark3-sql   --master yarn   --executor-memory 16G   --executor-cores 4   --driver-memory 16G   --conf spark.dynamicAllocation.maxExecutors=64

INSERT INTO wmf_dumps.wikitext_raw_rc1 SELECT * FROM xcollazo_iceberg. wikitext_raw_rc1 WHERE wiki_db='simplewiki' ORDER BY wiki_db, revision_timestamp;
...
Response code
Time taken: 220.514 seconds