How to get values from MySQL(5.6) column if that contains json document as string

前端未结

关注

 10  691

萌比男神i

How to get values from MySQL(5.6) column if that contains JSON document as a string

For example, if we have a table - employee in that we have three columns id, nam

相关标签:

10条回答

小蘑菇

2020-12-02 19:19

Here are my 3 SQL stored functions that I use for JSON extracting. They handle nested objects, but only care about the key name. The keys must be strings, the values can be strings, numeric or boolean values. Arrays are not treated well, only the first value is picked. They return NULL if no value found.

The first one, json_extract_1 picks only the first value if there are multiple keys with the same name. If you leave the LIMIT 1 clause, it throws 'Subquery returns more than 1 row' if more keys found (safe mode).

The second, json_extract_m collects all values with the same key in a comma-separated list.

The third one, json_extract_c is the slowest one, but it treats values with commas also correctly. Use it if it's absolutely necessary, e.g textual descriptions.

For all three the limitation is 999 keys. You can speed up if you prepare a table for the numbers subselect.

DELIMITER $$

/*
 * Single-value JSON extract - picks the first value
 */
DROP FUNCTION IF EXISTS `json_extract_1`$$
CREATE FUNCTION `json_extract_1`(json_txt TEXT, search_key VARCHAR (255)) 
    RETURNS TEXT
BEGIN
    RETURN (SELECT TRIM(SUBSTRING_INDEX(SUBSTRING_INDEX(SUBSTRING_INDEX(txt,':',-1), '"', 2), '"', -1)) AS val
    FROM (
        SELECT TRIM(SUBSTRING_INDEX(SUBSTRING_INDEX(SUBSTRING_INDEX( SUBSTRING_INDEX(json_txt , ',', n), ',',  -1 ), '}', 1), '{', -1)) AS txt
        FROM (SELECT t1.v + t2.v*10 + t3.v*100 AS n
            FROM (SELECT 0 AS v UNION ALL SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT 3 UNION ALL SELECT 4 UNION ALL SELECT 5 UNION ALL SELECT 6 UNION ALL SELECT 7 UNION ALL SELECT 8 UNION ALL SELECT 9) t1,
            (SELECT 0 AS v UNION ALL SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT 3 UNION ALL SELECT 4 UNION ALL SELECT 5 UNION ALL SELECT 6 UNION ALL SELECT 7 UNION ALL SELECT 8 UNION ALL SELECT 9) t2,
            (SELECT 0 AS v UNION ALL SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT 3 UNION ALL SELECT 4 UNION ALL SELECT 5 UNION ALL SELECT 6 UNION ALL SELECT 7 UNION ALL SELECT 8 UNION ALL SELECT 9) t3) numbers
        WHERE CHAR_LENGTH(json_txt ) - CHAR_LENGTH(REPLACE(json_txt , ',', '')) >= n - 1
        AND n>0 ) sp
    WHERE TRIM(SUBSTRING_INDEX(txt,':',1)) = CONCAT('"',search_key,'"')
    LIMIT 1   -- comment out for safe mode
    );
END$$

/*
 * Multi-value JSON extract - collects all values, group_concats them with comma
 */
DROP FUNCTION IF EXISTS `json_extract_m`$$
CREATE FUNCTION `json_extract_m`(json_txt TEXT, search_key VARCHAR (255)) 
    RETURNS TEXT
BEGIN
    RETURN (SELECT GROUP_CONCAT(TRIM(SUBSTRING_INDEX(SUBSTRING_INDEX(SUBSTRING_INDEX(txt,':',-1), '"', 2), '"', -1))) AS val
    FROM (
        SELECT TRIM(SUBSTRING_INDEX(SUBSTRING_INDEX(SUBSTRING_INDEX( SUBSTRING_INDEX(json_txt , ',', n), ',',  -1 ), '}', 1), '{', -1)) AS txt
        FROM (SELECT t1.v + t2.v*10 + t3.v*100 AS n
            FROM (SELECT 0 AS v UNION ALL SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT 3 UNION ALL SELECT 4 UNION ALL SELECT 5 UNION ALL SELECT 6 UNION ALL SELECT 7 UNION ALL SELECT 8 UNION ALL SELECT 9) t1,
            (SELECT 0 AS v UNION ALL SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT 3 UNION ALL SELECT 4 UNION ALL SELECT 5 UNION ALL SELECT 6 UNION ALL SELECT 7 UNION ALL SELECT 8 UNION ALL SELECT 9) t2,
            (SELECT 0 AS v UNION ALL SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT 3 UNION ALL SELECT 4 UNION ALL SELECT 5 UNION ALL SELECT 6 UNION ALL SELECT 7 UNION ALL SELECT 8 UNION ALL SELECT 9) t3) numbers
        WHERE CHAR_LENGTH(json_txt ) - CHAR_LENGTH(REPLACE(json_txt , ',', '')) >= n - 1
        AND n>0 ) sp
    WHERE TRIM(SUBSTRING_INDEX(txt,':',1)) = CONCAT('"',search_key,'"'));
END$$

/*
 * Comma-safe JSON extract - treats values with commas correctly, but slow like hell
 */
DROP FUNCTION IF EXISTS `json_extract_c`$$
CREATE FUNCTION `json_extract_c`(json_txt TEXT, search_key VARCHAR (255)) 
    RETURNS TEXT
BEGIN
    DROP TEMPORARY TABLE IF EXISTS json_parts;
    DROP TEMPORARY TABLE IF EXISTS json_parts2;
    DROP TEMPORARY TABLE IF EXISTS json_indexes;

    CREATE TEMPORARY TABLE json_parts AS
    SELECT n, IF(INSTR(txt,':')>0 AND (INSTR(txt,',')+INSTR(txt,'{')>0),1,0) AS this_val, IF(INSTR(txt,':')>0 AND (INSTR(txt,',')+INSTR(txt,'{')=0),1,0) AS next_val, IF(INSTR(txt,',')+INSTR(txt,'{')>0,1,0) AS next_key, txt
    FROM (SELECT n, IF(n%2,txt,REPLACE(txt,',','|')) AS txt 
    FROM (SELECT n, TRIM(SUBSTRING_INDEX(SUBSTRING_INDEX(json_txt , '"', n), '"',  -1 )) AS txt
        FROM (SELECT t1.v + t2.v*10 + t3.v*100 AS n
        FROM (SELECT 0 AS v UNION ALL SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT 3 UNION ALL SELECT 4 UNION ALL SELECT 5 UNION ALL SELECT 6 UNION ALL SELECT 7 UNION ALL SELECT 8 UNION ALL SELECT 9) t1,
        (SELECT 0 AS v UNION ALL SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT 3 UNION ALL SELECT 4 UNION ALL SELECT 5 UNION ALL SELECT 6 UNION ALL SELECT 7 UNION ALL SELECT 8 UNION ALL SELECT 9) t2,
        (SELECT 0 AS v UNION ALL SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT 3 UNION ALL SELECT 4 UNION ALL SELECT 5 UNION ALL SELECT 6 UNION ALL SELECT 7 UNION ALL SELECT 8 UNION ALL SELECT 9) t3) numbers
        WHERE CHAR_LENGTH(json_txt ) - CHAR_LENGTH(REPLACE(json_txt , '"', '')) >= n - 1
        AND n>0) v
        ) v2;
    CREATE TEMPORARY TABLE json_parts2 AS SELECT * FROM json_parts;

    CREATE TEMPORARY TABLE json_indexes AS
    SELECT p1.n, p1.n+1 AS key_idx, MIN(GREATEST(IF(p2.this_val,p2.n,0), IF(p2.next_val,p2.n+1,0))) AS val_idx, p2.this_val AS trim_val
    FROM json_parts p1
    JOIN json_parts2 p2 ON (p1.n < p2.n AND (p2.this_val OR p2.next_val)) 
    WHERE p1.next_key
    GROUP BY p1.n;  

    RETURN (SELECT json_values.v 
        FROM (SELECT p1.txt AS k, REPLACE(IF(i.trim_val, regexp_replace(regexp_replace(p2.txt,'^[: {]+',''),'[, }]+$',''), p2.txt), '|', ',') AS v
            FROM json_indexes i
            JOIN json_parts p1 ON (i.key_idx = p1.n)
            JOIN json_parts2 p2 ON (i.val_idx = p2.n)) json_values
        WHERE json_values.k = search_key);
END$$

DELIMITER ;

Yep, and if you have the chance, try to upgrade to MySQL 5.7, the built-in functions work much more efficiently.

0 讨论(0)

没有蜡笔的小新

2020-12-02 19:23

Yes , you can definitely to it using JSON_EXTRACT() function in mysql.

lets take a table that contains JSON (table client_services here) :

+-----+-----------+--------------------------------------+
| id  | client_id | service_values                       |
+-----+-----------+------------+-------------------------+
| 100 |      1000 | { "quota": 1,"data_transfer":160000} |
| 101 |      1000 | { "quota": 2,"data_transfer":800000} |
| 102 |      1000 | { "quota": 3,"data_transfer":70000}  |
| 103 |      1001 | { "quota": 1,"data_transfer":97000}  |
| 104 |      1001 | { "quota": 2,"data_transfer":1760}   |
| 105 |      1002 | { "quota": 2,"data_transfer":1060}   |
+-----+-----------+--------------------------------------+

To Select each JSON fields , run this query :

SELECT 
    id, client_id, 
    json_extract(service_values, '$.quota') AS quota,
    json_extract(service_values, '$.data_transfer') AS data_transfer
FROM client_services;

So the output will be :

+-----+-----------+----------------------+
| id  | client_id | quota | data_transfer|
+-----+-----------+----------------------+
| 100 |      1000 |     1 |       160000 |
| 101 |      1000 |     2 |       800000 |
| 102 |      1000 |     3 |        70000 |
| 103 |      1001 |     1 |        97000 |
| 104 |      1001 |     2 |         1760 |
| 105 |      1002 |     2 |         1060 |
+-----+-----------+----------------------+

Hope this solves your problem!

0 讨论(0)

北荒

2020-12-02 19:33

Function above not work properly if you have nested JSON in table field.

Because i needed a JSON_EXTRACT on mysql 5.6 i wrote it by myself a copy of original function that can extract values like the native function in mysql 5.7

Usage:

SELECT JSON_EXTRACT_NESTED(table_field,"json_level1.json_level2.json_level3") FROM table;

If you have one level JSON then you use:

SELECT JSON_EXTRACT_NESTED(table_field,"json_level1") FROM table;

In database you have to add two functions:

Main function:

CREATE FUNCTION `json_extract_nested`(
_field TEXT,
_variable TEXT
) RETURNS TEXT CHARSET latin1
BEGIN
                DECLARE X INT DEFAULT 0;
                DECLARE fieldval1 TEXT;
                DECLARE arrayName,arrayValue TEXT;

                SET arrayName = SUBSTRING_INDEX(_variable, '.', 1);

                IF(LOCATE('%',arrayName)> 0) THEN 
                    SET _field = SUBSTRING_INDEX(_field, "{", -1);
                    SET _field = SUBSTRING_INDEX(_field, "}", 1);
                    RETURN TRIM(
                    BOTH '"' FROM SUBSTRING_INDEX(
                        SUBSTRING_INDEX(
                        SUBSTRING_INDEX(
                            _field,
                            CONCAT(
                            '"',
                            SUBSTRING_INDEX(_variable,'$.', - 1),
                            '":'
                            ),
                            - 1
                        ),
                        ',"',
                        1
                        ),
                        ':',
                        -1
                    )
                    ) ;
                ELSE  
                    SET arrayValue = json_array_value(_field, arrayName);
                    WHILE X < (LENGTH(_variable) - LENGTH(REPLACE(_variable, '.', ""))) DO
                        IF(LENGTH(_variable) - LENGTH(REPLACE(_variable, '.', ""))>X) THEN
                            SET arrayName = SUBSTRING_INDEX(SUBSTRING_INDEX(_variable, '.', X+2),'.',-1);
                        END IF;
                        IF(arrayName<>'') THEN
                            SET arrayValue = json_array_value(arrayValue, arrayName);
                        END IF;
                        SET X = X + 1;
                    END WHILE;
                END IF;
                RETURN arrayValue;
                END$$
DELIMITER ;

Auxiliary function (needed by main function):

CREATE FUNCTION `json_array_value`(
  _field  TEXT,
  arrayName VARCHAR (255)
) RETURNS TEXT CHARSET latin1
BEGIN
                DECLARE arrayValue, arrayValueTillDelimit TEXT;
                DECLARE arrayStartDelimiter, arrayEndDelimiter VARCHAR(10);
                DECLARE arrayCountDelimiter INT;
                DECLARE countBracketLeft, countBracketRight INT DEFAULT 0;
                DECLARE X INT DEFAULT 0;
                DECLARE arrayNameQuoted VARCHAR(255);
                SET arrayNameQuoted = CONCAT('"',arrayName,'"');
                /*check arrayname exist*/
                IF(LOCATE(arrayNameQuoted,_field)= 0) THEN 
                    RETURN NULL;    
                ELSE
                    /*get value behind arrayName1*/
                    SET _field = SUBSTRING(_field,1,LENGTH(_field)-1);
                    SET arrayValue = SUBSTRING(_field, LOCATE(arrayNameQuoted,_field)+LENGTH(arrayNameQuoted)+1, LENGTH(_field));
                    /*get json delimiter*/
                    SET arrayStartDelimiter = LEFT(arrayValue, 1);
                    IF(arrayStartDelimiter='{') THEN
                        SET arrayEndDelimiter = '}';                            
                        loopBrackets: WHILE X < (LENGTH(arrayValue)) DO                 
                            SET countBracketLeft = countBracketLeft +IF(SUBSTRING(arrayValue,X,1)=arrayStartDelimiter,1,0);
                        SET countBracketRight = countBracketRight +IF(SUBSTRING(arrayValue,X,1)=arrayEndDelimiter,1,0);
                        IF(countBracketLeft<>0 AND countBracketLeft=countBracketRight) THEN
                            SET arrayCountDelimiter = X;
                            LEAVE loopBrackets;
                        ELSE
                            SET X = X + 1;
                        END IF;
                        END WHILE;
                                ELSEIF(arrayStartDelimiter='[') THEN
                                    SET arrayEndDelimiter = ']';
                                    SET arrayCountDelimiter = LENGTH(SUBSTRING_INDEX(arrayValue, arrayEndDelimiter, 0));
                                 ELSEIF(arrayStartDelimiter='"') THEN
                                    SET arrayEndDelimiter = '"';
                                    SET arrayCountDelimiter = LENGTH(SUBSTRING_INDEX(arrayValue, arrayEndDelimiter, 0));
                                ELSE 
                                    SET arrayStartDelimiter = "";
                                    IF((LOCATE(",",arrayValue)> LOCATE("}",arrayValue))) THEN
                        SET arrayEndDelimiter = ",";
                        ELSE
                        SET arrayEndDelimiter = "}";
                        END IF;
                        SET arrayCountDelimiter = LENGTH(SUBSTRING_INDEX(arrayValue, arrayEndDelimiter, 0));
                    END IF;
                    SET arrayValueTillDelimit = SUBSTRING(arrayValue, 1, arrayCountDelimiter);
                    SET arrayCountDelimiter = LENGTH(arrayValueTillDelimit) - LENGTH(REPLACE(arrayValueTillDelimit, arrayStartDelimiter, ""));
                    SET arrayValue = SUBSTR(arrayValue,LENGTH(arrayStartDelimiter)+1);
                    IF(arrayStartDelimiter='{') THEN
                        SET arrayValue = SUBSTRING_INDEX(arrayValue, arrayEndDelimiter, arrayCountDelimiter);
                    ELSE
                        SET arrayValue = SUBSTRING_INDEX(arrayValue, arrayEndDelimiter, arrayCountDelimiter+1);
                    END IF;
                    RETURN (arrayValue);
                END IF;
                END$$
DELIMITER ;

0 讨论(0)

鱼传尺愫

2020-12-02 19:35

Offering this alternative view of the answers given here for those of you (like me) who may not intuitively see the string manipulation within the SQL functions. This version will allow you to explicitly see each step of the text parsing. This works for MySQL 5.6 and can of course be combined back together and not use any variables.

DELIMITER $$ 
DROP FUNCTION IF EXISTS `json_extract_c`$$
CREATE FUNCTION `json_extract_c`(
  details TEXT,
  required_field VARCHAR (255)
) RETURNS TEXT CHARSET latin1
BEGIN
  /* get key from function passed required field value */
  set @JSON_key = SUBSTRING_INDEX(required_field,'$.', -1); 
  /* get everything to the right of the 'key = <required_field>' */
  set @JSON_entry = SUBSTRING_INDEX(details,CONCAT('"', @JSON_key, '"'), -1 ); 
  /* get everything to the left of the trailing comma */
  set @JSON_entry_no_trailing_comma = SUBSTRING_INDEX(@JSON_entry, ",", 1); 
  /* get everything to the right of the leading colon after trimming trailing and leading whitespace */
  set @JSON_entry_no_leading_colon = TRIM(LEADING ':' FROM TRIM(@JSON_entry_no_trailing_comma)); 
  /* trim off the leading and trailing double quotes after trimming trailing and leading whitespace*/
  set @JSON_extracted_entry = TRIM(BOTH '"' FROM TRIM(@JSON_entry_no_leading_colon));
  RETURN @JSON_extracted_entry;
END$$
DELIMITER ;

0 讨论(0)

隐瞒了意图╮

2020-12-02 19:36

Rahul's answer did not work quite well for me, so I edited it and this worked for me:

DELIMITER $$

DROP FUNCTION IF EXISTS `json_extract_c`$$

CREATE FUNCTION `json_extract_c`(
details TEXT,
required_field VARCHAR (255)
) RETURNS TEXT CHARSET latin1
BEGIN
SET details = TRIM(LEADING '{' FROM TRIM(details));
SET details = TRIM(TRAILING '}' FROM TRIM(details));
RETURN TRIM(
    BOTH '"' FROM SUBSTRING_INDEX(
        SUBSTRING_INDEX(
            SUBSTRING_INDEX(
                details,
                CONCAT(
                    '"',
                    SUBSTRING_INDEX(required_field,'$.', - 1),
                    '":'
                ),
                - 1
            ),
            ',"',
            1
        ),
        ':',
        -1
    )
) ;
END$$

DELIMITER ;

0 讨论(0)

别跟我提以往

2020-12-02 19:37

This is our deployed json_extract_c in MySQL 5.6

Tested:

select json_extract_c('{"a": 1, "b": 2}', "$.a"); -> 1;
select json_extract_c('{"a": "1", "b": "2"}', "$.a"); -> 1;
select json_extract_c('{"a":"1","b":"2"}', "$.a"); -> 1;
select json_extract_c('{"a"    :"1",  "b" :"2"}', "$.a"); -> 1;
select json_extract_c('{"b"    :"a",  "a" :"2"}', "$.a"); -> 2;
select json_extract_c('{"a"    : "a",  "a" :"2"}', "$.a"); -> a;
select json_extract_c('{"a": "1"}', "$.a"); -> 1
select json_extract_c('{"a": "a"}', "$.a"); -> a
select json_extract_c('{"a"   : "a"}', "$.a"); -> a
select json_extract_c('{"a.a"   : "a"}', "$.a"); -> NULL
select json_extract_c('{"a\"a"   : "9"}', "$.a"); -> NULL

Not supported:

Nested json
Escaped double quotes in json values, e.g. select json_extract_c('{"a\"a" : "9", "a" : "a\"a"}', "$.a"); -> a

Limitation:

Only first field will be extracted if there are multiple same fields, e.g. select json_extract_c('{"a" : 1, "a" : "2}', "$.a"); -> 1

DELIMITER $$
DROP function if exists json_extract_c$$
CREATE FUNCTION json_extract_c(
details TEXT,
required_field VARCHAR (255)
) RETURNS TEXT CHARSET utf8mb4
DETERMINISTIC
NO SQL
BEGIN
  DECLARE search_term, val TEXT;
  DECLARE pos INT signed DEFAULT 1;

  -- Remove '{' and '}'
  SET details = SUBSTRING_INDEX(details, "{", -1);
  SET details = SUBSTRING_INDEX(details, "}", 1);
  -- Transform '$.xx' to be '"xx"'
  SET search_term = CONCAT('"', SUBSTRING_INDEX(required_field,'$.', - 1), '"');

  searching: LOOP
    SET pos = LOCATE(search_term, details);
    -- Keep searching if the field contains escape chars
    WHILE pos > 0 AND RIGHT(LEFT(details, pos-1), 1) = '\\'
    DO
      SET details = SUBSTR(details, pos+LENGTH(search_term));
      SET pos = LOCATE(search_term, details);
    END WHILE;
    -- Return NULL if not found
    IF pos <= 0 THEN
      RETURN NULL;
    END IF;

    SET pos = LENGTH(search_term)+pos;
    SET details = SUBSTR(details, pos);
    SET val = TRIM(details);

    -- see if we reach the value that is a leading colon ':'
    IF LEFT(val, 1) = ':' THEN
      RETURN TRIM(
        TRAILING ',' FROM 
        TRIM(
          SUBSTRING_INDEX(
            TRIM(
              BOTH '"' FROM TRIM(
                SUBSTR(
                  val
                , 2
                )
              )
            )
          , '"', 1
          )
        )
      );
    ELSE
      ITERATE searching;
    END IF;
  END LOOP;
END$$
DELIMITER ;

0 讨论(0)

1 2 下一页