#!/usr/bin/env python3
# Copyright (C) 2023 Viktor Soderqvist <viktor dot soderqvist at est dot tech>
# This file is released under the BSD license, see the COPYING file
# This script generates cmddef.h from the JSON files in the Redis repo
# describing the commands. This is done manually when commands have been added
# to Redis or when you want add more commands implemented in modules, etc.
#
# Usage: ./gencommands.py path/to/redis/src/commands/*.json > cmddef.h
#
# Alternatively, the output of the script utils/generate-commands-json.py (which
# fetches the command metadata from a running Redis node) or the file
# commands.json from the redis-doc repo can be used as input to this script:
# https://github.com/redis/redis-doc/blob/master/commands.json
#
# Additional JSON files can be added to extend support for custom commands. The
# JSON file format is not fully documented but hopefully the format can be
# understood from reading the existing JSON files. Alternatively, you can read
# the source code of this script to see what it does.
#
# The key specifications part is documented here:
# https://redis.io/docs/reference/key-specs/
#
# The discussion where this JSON format was added in Redis is here:
# https://github.com/redis/redis/issues/9359
#
# For convenience, files on the output format like cmddef.h can also be used as
# input files to this script. It can be used for adding more commands to the
# existing set of commands, but please do not abuse it. Do not to write commands
# information directly in this format.
import glob
import json
import os
import sys
import re
# Returns True if any of the nested arguments is a key; False otherwise.
def any_argument_is_key(arguments):
for arg in arguments:
if arg.get("type") == "key":
return True
if "arguments" in arg and any_argument_is_key(arg["arguments"]):
return True
return False
# Returns a tuple (method, index) where method is one of the following:
#
# NONE = No keys
# UNKNOWN = The position of the first key is unknown or too
# complex to describe (example XREAD)
# INDEX = The first key is the argument at index i
# KEYNUM = The argument at index i specifies the number of keys
# and the first key is the next argument, unless the
# number of keys is zero in which case there are no
# keys (example EVAL)
def firstkey(props):
if not "key_specs" in props:
# Key specs missing. Best-effort fallback to "arguments".
if "arguments" in props:
args = props["arguments"]
for i in range(1, len(args)):
arg = args[i - 1]
if arg.get("type") == "key":
return ("INDEX", i)
elif arg.get("type") == "string" and arg.get("name") == "key":
# add-hoc case for RediSearch
return ("INDEX", i)
elif arg.get("optional") or arg.get("multiple") or "arguments" in arg:
# Too complex for this fallback.
if any_argument_is_key(args):
return ("UNKNOWN", 0)
else:
return ("NONE", 0)
return ("NONE", 0)
if len(props["key_specs"]) == 0:
return ("NONE", 0)
# We detect the first key spec and only if the begin_search is by index.
# Otherwise we return -1 for unknown (for example if the first key is
# indicated by a keyword like KEYS or STREAMS).
begin_search = props["key_specs"][0]["begin_search"]
if "index" in begin_search:
# Redis source JSON files have this syntax
pos = begin_search["index"]["pos"]
elif begin_search.get("type") == "index" and "spec" in begin_search:
# generate-commands-json.py returns this syntax
pos = begin_search["spec"]["index"]
else:
return ("UNKNOWN", 0)
find_keys = props["key_specs"][0]["find_keys"]
if "range" in find_keys or find_keys.get("type") == "range":
# The first key is the arg at index pos.
# Redis source JSON files have this syntax:
# "find_keys": {
# "range": {...}
# }
# generate-commands-json.py returns this syntax:
# "find_keys": {
# "type": "range",
# "spec": {...}
# },
return ("INDEX", pos)
elif "keynum" in find_keys:
# The arg at pos is the number of keys and the next arg is the first key
# Redis source JSON files have this syntax
assert find_keys["keynum"]["keynumidx"] == 0
assert find_keys["keynum"]["firstkey"] == 1
return ("KEYNUM", pos)
elif find_keys.get("type") == "keynum":
# generate-commands-json.py returns this syntax
assert find_keys["spec"]["keynumidx"] == 0
assert find_keys["spec"]["firstkey"] == 1
return ("KEYNUM", pos)
else:
return ("UNKNOWN", 0)
def extract_command_info(name, props):
(firstkeymethod, firstkeypos) = firstkey(props)
container = props.get("container", "")
name = name.upper()
subcommand = None
if container != "":
subcommand = name
name = container.upper()
else:
# Ad-hoc handling of command and subcommand in the same string,
# sepatated by a space. This form is used in e.g. RediSearch's JSON file
# in commands like "FT.CONFIG GET".
tokens = name.split(maxsplit=1)
if len(tokens) > 1:
name, subcommand = tokens
if firstkeypos > 0 and not "key_specs" in props:
# Position was inferred from "arguments"
firstkeypos += 1
arity = props["arity"] if "arity" in props else -1
return (name, subcommand, arity, firstkeymethod, firstkeypos);
# Parses a file with lines like
# COMMAND(identifier, cmd, subcmd, arity, firstkeymethod, firstkeypos)
def collect_command_from_cmddef_h(f, commands):
for line in f:
m = re.match(r'^COMMAND\(\S+, *"(\S+)", NULL, *(-?\d+), *(\w+), *(\d+)\)', line)
if m:
commands[m.group(1)] = (m.group(1), None, int(m.group(2)), m.group(3), int(m.group(4)))
continue
m = re.match(r'^COMMAND\(\S+, *"(\S+)", *"(\S+)", *(-?\d+), *(\w+), *(\d)\)', line)
if m:
key = m.group(1) + "_" + m.group(2)
commands[key] = (m.group(1), m.group(2), int(m.group(3)), m.group(4), int(m.group(5)))
continue
if re.match(r'^(?:/\*.*\*/)?\s*$', line):
# Comment or blank line
continue
else:
print("Error processing line: %s" % (line))
exit(1)
def collect_commands_from_files(filenames):
# The keys in the dicts are "command" or "command_subcommand".
commands = dict()
commands_that_have_subcommands = set()
for filename in filenames:
with open(filename, "r") as f:
if filename.endswith(".h"):
collect_command_from_cmddef_h(f, commands)
continue
try:
d = json.load(f)
for name, props in d.items():
cmd = extract_command_info(name, props)
(name, subcmd, _, _, _) = cmd
# For commands with subcommands, we want only the
# command-subcommand pairs, not the container command alone
if subcmd is not None:
commands_that_have_subcommands.add(name)
if name in commands:
del commands[name]
name += "_" + subcmd
elif name in commands_that_have_subcommands:
continue
commands[name] = cmd
except json.decoder.JSONDecodeError as err:
print("Error processing %s: %s" % (filename, err))
exit(1)
return commands
def generate_c_code(commands):
print("/* This file was generated using gencommands.py */")
print("")
print("/* clang-format off */")
for key in sorted(commands):
(name, subcmd, arity, firstkeymethod, firstkeypos) = commands[key]
# Make valid C identifier (macro name)
key = re.sub(r'\W', '_', key)
if subcmd is None:
print("COMMAND(%s, \"%s\", NULL, %d, %s, %d)" %
(key, name, arity, firstkeymethod, firstkeypos))
else:
print("COMMAND(%s, \"%s\", \"%s\", %d, %s, %d)" %
(key, name, subcmd, arity, firstkeymethod, firstkeypos))
# MAIN
if len(sys.argv) < 2 or sys.argv[1] == "--help":
print("Usage: %s path/to/redis/src/commands/*.json > cmddef.h" % sys.argv[0])
exit(1)
# Find all JSON files
filenames = []
for filename in sys.argv[1:]:
if os.path.isdir(filename):
# A redis repo root dir (accepted for backward compatibility)
jsondir = os.path.join(filename, "src", "commands")
if not os.path.isdir(jsondir):
print("The directory %s is not a Redis source directory." % filename)
exit(1)
filenames += glob.glob(os.path.join(jsondir, "*.json"))
else:
filenames.append(filename)
# Collect all command info
commands = collect_commands_from_files(filenames)
# Print C code
generate_c_code(commands)