Page MenuHomePhabricator

"Column 'type' cannot be null"
Closed, ResolvedPublicBUG REPORT

Description

What is the problem?

Some combinations of data causes the below exception.

/srv/service/node_modules/mariadb/lib/misc/errors.js:61
  return new SqlError(msg, sql, fatal, info, sqlState, errno, additionalStack, addHeader);
         ^

SqlError: (conn=1185, no: 1048, SQLState: 23000) Column 'type' cannot be null
sql: INSERT INTO
                        tunnels (operator, type, anonymous)
                    VALUES
                        (?, ?, ?) - parameters:['ó5Ýoð𝾿#󽡐',null,-4620852125366381000]
    at Object.module.exports.createError (/srv/service/node_modules/mariadb/lib/misc/errors.js:61:10)
    at PacketNodeEncoded.readError (/srv/service/node_modules/mariadb/lib/io/packet.js:511:19)
    at Query.readResponsePacket (/srv/service/node_modules/mariadb/lib/cmd/resultset.js:46:28)
    at PacketInputStream.receivePacketBasic (/srv/service/node_modules/mariadb/lib/io/packet-input-stream.js:104:9)
    at PacketInputStream.onData (/srv/service/node_modules/mariadb/lib/io/packet-input-stream.js:169:20)
    at Socket.emit (node:events:513:28)
    at addChunk (node:internal/streams/readable:315:12)
    at readableAddChunk (node:internal/streams/readable:289:9)
    at Socket.Readable.push (node:internal/streams/readable:228:10)
    at TCP.onStreamRead (node:internal/stream_base_commons:190:23) {
  text: "Column 'type' cannot be null",
  sql: 'INSERT INTO\n' +
    '                        tunnels (operator, type, anonymous)\n' +
    '                    VALUES\n' +
    "                        (?, ?, ?) - parameters:['ó5Ýoð\x04𝾿#󽡐',null,-4620852125366381000]",
  fatal: false,
  errno: 1048,
  sqlState: '23000',
  code: 'ER_BAD_NULL_ERROR'
}
Steps to reproduce problem
  1. Save the JSON from "Reproduction data" below as a .gz file (e.g. type_null_reprod.json.gz) into the ipoid/tmp directory
  2. If necessary, start up docker in the ipoid directory (e.g. docker compose up -d)
  3. Initialise the database: docker compose exec web node init-db.js
  4. Run this command twice: docker compose exec -e FEED_PATH=./tmp/type_null_reprod.json.gz web node import-data.js
    • (Note, the first time you run that command you will probably see the error in T339325)
Environment

ipoid commit deed45d61626f952ce8b9365e12e8bd98703fa17

Reproduction data
{"as": {"Organization": null, "number": null}, "client": {"behaviors": ["P"], "concentration": {"city": false, "density": null}, "count": null, "countries": null, "proxies": [true, "", false, true], "spread": "5", "types": ["DESKTOP", "IOT", "IOT", "DESKTOP", "IOT"]}, "infrastructure": null, "location": {"city": 14298}, "organization": "", "risks": ["WEB_SCRAPING", "CALLBACK_PROXY", "CALLBACK_PROXY", "LOGIN_BRUTEFORCE", "CALLBACK_PROXY"], "services": [null, "\u0081\u00ed", false, "\u00fe%", "j", true, "E", "\u0092%", "\ud802\ude3d", "\u00f3", "\u00c9\u00a4\udb66\udc9cs\n", 25195, null], "tunnels": [{"anonymous": -4620852125366380810, "entries": [-3511109048614173750, null, null, "\udb4a\udd40", 31887, "\u00b5E \ud9dd\udf48t\u00c5", null, false, "N", true, 18166, -4252, -16176], "operator": "\u00f35\u00ddo\u00f0\u0004\ud837\udfbf#\udbb6\udc50", "type": null}, {"anonymous": "\u00d2", "entries": [4107, true, "z", "\uda9f\udf45", null, 76, -488428306024344228, "\u00bb", 1023, null, "\u00b7", "\u001cV\uda9a\udc61\u00b1\ud88b\udd3c\u00cb\ud9ac\udcfc\u5bf8n\ud920\udf85\u00bf\b\ud957\udfa3\u00ef", null, "\ud908\udc37", ">\udaa7\udc88\u00d6\u00cf\uf94c\u00b3\u008a", "\u00ccd", 32701], "operator": "\u0012", "type": null}, {"anonymous": -28954, "entries": [17904, false, null, "\udb81\ude45"], "operator": 11815, "type": null}, {"anonymous": false, "entries": ["\ud968\udf3e", 126, -7858, "\u008d", null, false], "operator": false, "type": 87}, {"anonymous": null, "entries": ["\u0083\ud955\ude3d\u00fd", "e", "*"], "operator": true, "type": "\u009e"}], "ip": "253dce88-bd35-4898-8ba9-a80cbb4c6dca"}
{"as": {}, "client": {"behaviors": [], "concentration": {"density": "\u0089"}, "count": -108, "countries": null, "proxies": [null, "l", null, false, "\u0083"], "spread": 56, "types": ["IOT", "MOBILE", "IOT"]}, "infrastructure": 1565, "location": {"country": false}, "organization": -107, "risks": ["LOGIN_BRUTEFORCE", "CALLBACK_PROXY", "TUNNEL", "WEB_SCRAPING", "WEB_SCRAPING", "GEO_MISMATCH", "LOGIN_BRUTEFORCE", "WEB_SCRAPING", "WEB_SCRAPING", "WEB_SCRAPING", "WEB_SCRAPING", "WEB_SCRAPING", "LOGIN_BRUTEFORCE"], "services": [120, null, false, true, true, "\u00f7+>\u00a0_g"], "tunnels": [{"anonymous": false, "entries": ["\u000f\u008c\ud93e\udfca", "\u00d8\ud949\ude4d\u00cc\u00cc,\u00d0\u00d8\u0019\u00ae\uda82\udc04\u00bc!\u00da\u00e7IT\u00d7", 6356430817251319490], "operator": "\u00cd", "type": "\u0017\ub96e5"}, {"anonymous": "\u000b", "entries": [null, null, "|", 83, true, null, "e", true, "\udb84\udec7[6\ud91e\udd7f9\ud8b0\ude02", true, null, "V5\u009b'\u00d1\udb89\ude50\udaba\uddd8\u000b\u00c5\u0006\ud8a4\udc0a\u00b4\u00f9\udb3f\udf72\u00d8\u009c", null, -20446], "operator": "B\u0015\u0018\ud87f\udddbG", "type": "\u009f"}, {"anonymous": -4882, "entries": [29490, true], "operator": 121, "type": "\u00a5"}, {"anonymous": false, "entries": [], "operator": -15, "type": "\udb85\udea2"}, {"anonymous": -6339, "entries": [true, true, false, "G", 582], "operator": false, "type": null}], "ip": "2a6bb956-708f-4b6a-8e16-bca86ea33f8d"}
{"as": {"Organization": "k", "number": "\u00e6\ud86a\udf8e\udbbb\uddaeT"}, "client": {"behaviors": [true, 7603, "", "\u008f", -103, -15500, -21], "concentration": {}, "count": 8887, "countries": null, "proxies": [-27079], "spread": 17498, "types": ["DESKTOP", "DESKTOP", "IOT", "IOT", "DESKTOP", "IOT", "HEADLESS", "DESKTOP", "DESKTOP", "MOBILE"]}, "infrastructure": true, "location": {"country": ""}, "organization": 33647638431791341989110704120025675892, "risks": ["TUNNEL", "CALLBACK_PROXY", "WEB_SCRAPING", "GEO_MISMATCH", "WEB_SCRAPING", "LOGIN_BRUTEFORCE", "WEB_SCRAPING"], "services": [], "tunnels": [{"anonymous": true, "entries": [-9363, "\ud954\udce4"], "operator": "h\uda0e\udc39", "type": "g"}, {"anonymous": null, "entries": [null, 1165], "operator": "Z\u00a1", "type": 17953}], "ip": "0e7c4b67-d1b5-48e5-b92a-3d36c5794feb"}

Event Timeline

This should be covered by this merge request https://gitlab.wikimedia.org/repos/mediawiki/services/ipoid/-/merge_requests/27
The schema has been updated to accept null on this merge request.
I will move this to along with T339359

cc: @dom_walden/ @GMikesell-WMF

This should be covered by this merge request https://gitlab.wikimedia.org/repos/mediawiki/services/ipoid/-/merge_requests/27
The schema has been updated to accept null on this merge request.
I will move this to along with T339359

cc: @dom_walden/ @GMikesell-WMF

Confirming the above while testing T339359
Without the schema change, you get the Not Null error which is fixed in Tsepo's patch.

@TThoabala As discussed, I will close this task since we are getting a different issue with the latest patch beside code: 'ER_BAD_NULL_ERROR'. I created T341605: Ipoid: ER_DATA_TOO_LONG. Thanks!