A proper send_string() for the Unicode feature (#8155)

* Add mouse support to SEND_STRING

* add short forms

docs/cli.md

@@ -71,14 +71,36 @@ There are some limitations to the local CLI compared to the global CLI:
 
 ## `qmk cformat`
 
-This command formats C code using clang-format. Run it with no arguments to format all core code, or pass filenames on the command line to run it on specific files.
+This command formats C code using clang-format. 
 
-**Usage**:
+Run it with no arguments to format all core code that has been changed. Default checks `origin/master` with `git diff`, branch can be changed using `-b <branch_name>`
+
+Run it with `-a` to format all core code, or pass filenames on the command line to run it on specific files.
+
+**Usage for specified files**:
 
 ```
 qmk cformat [file1] [file2] [...] [fileN]
 ```
 
+**Usage for all core files**:
+
+```
+qmk cformat -a
+```
+
+**Usage for only changed files against origin/master**:
+
+```
+qmk cformat
+```
+
+**Usage for only changed files against branch_name**:
+
+```
+qmk cformat -b branch_name
+```
+
 ## `qmk compile`
 
 This command allows you to compile firmware from any directory. You can compile JSON exports from <https://config.qmk.fm>, compile keymaps in the repo, or compile the keyboard in the current working directory.

docs/feature_macros.md

@@ -154,6 +154,8 @@ SEND_STRING(".."SS_TAP(X_END));
 
 There are some functions you may find useful in macro-writing. Keep in mind that while you can write some fairly advanced code within a macro, if your functionality gets too complex you may want to define a custom keycode instead. Macros are meant to be simple.
 
+?> You can also use the functions described in [Useful function](ref_functions.md) for additional functionality. For example `reset_keyboard()` allows you to reset the keyboard as part of a macro.
+
 ### `record->event.pressed`
 
 This is a boolean value that can be tested to see if the switch is being pressed or released. An example of this is

docs/feature_unicode.md

@@ -193,12 +193,24 @@ By default, when the keyboard boots, it will initialize the input mode to the la
 
 !> Using `UNICODE_SELECTED_MODES` means you don't have to initially set the input mode in `matrix_init_user()` (or a similar function); the Unicode system will do that for you on startup. This has the added benefit of avoiding unnecessary writes to EEPROM.
 
-## `send_unicode_hex_string`
+## `send_unicode_string()`
 
-To type multiple characters for things like (ノಠ痊ಠ)ノ彡┻━┻, you can use `send_unicode_hex_string()` much like `SEND_STRING()` except you would use hex values separate by spaces.
+This function is much like `send_string()` but allows you to input UTF-8 characters directly, currently up to code point U+FFFF. Make sure your `keymap.c` is formatted in UTF-8 encoding.
-For example, the table flip seen above would be `send_unicode_hex_string("0028 30CE 0CA0 75CA 0CA0 0029 30CE 5F61 253B 2501 253B")`
 
-There are many ways to get a hex code, but an easy one is [this site](https://r12a.github.io/app-conversion/). Just make sure to convert to hexadecimal, and that is your string.
+```c
+send_unicode_string("(ノಠ痊ಠ)ノ彡┻━┻");
+```
+
+## `send_unicode_hex_string()`
+
+Similar to `send_unicode_string()`, but the characters are represented by their code point values in ASCII, separated by spaces. For example, the table flip above would be achieved with:
+
+```c
+send_unicode_hex_string("0028 30CE 0CA0 75CA 0CA0 0029 30CE 5F61 253B 2501 253B");
+```
+
+An easy way to convert your Unicode string to this format is by using [this site](https://r12a.github.io/app-conversion/), and taking the result in the "Hex/UTF-32" section.
+Unlike `send_unicode_string()` this function supports code points up to U+10FFFF.
 
 ## Additional Language Support
 
@@ -228,6 +240,6 @@ AutoHotkey inserts the Text right of `Send, ` when this combination is pressed.
 
 If you enable the US International layout on the system, it will use punctuation to accent the characters.
 
-For instance, typing "`a" will result in à.
+For instance, typing "\`a" will result in à.
 
 You can find details on how to enable this [here](https://support.microsoft.com/en-us/help/17424/windows-change-keyboard-layout).

drivers/arm/i2c_master.c

@@ -29,6 +29,8 @@
 #include <string.h>
 #include <hal.h>
 
+static uint8_t i2c_address;
+
 static const I2CConfig i2cconfig = {
 #ifdef USE_I2CV1
     I2C1_OPMODE,
@@ -69,49 +71,27 @@ __attribute__((weak)) void i2c_init(void) {
 }
 
 i2c_status_t i2c_start(uint8_t address) {
-#if I2C_USE_MUTUAL_EXCLUSION
+    i2c_address = address;
-    i2cAcquireBus(&I2C_DRIVER);
-#endif
-
     i2cStart(&I2C_DRIVER, &i2cconfig);
     return I2C_STATUS_SUCCESS;
 }
 
 i2c_status_t i2c_transmit(uint8_t address, const uint8_t* data, uint16_t length, uint16_t timeout) {
-#if I2C_USE_MUTUAL_EXCLUSION
+    i2c_address = address;
-    i2cAcquireBus(&I2C_DRIVER);
-#endif
-
     i2cStart(&I2C_DRIVER, &i2cconfig);
-    msg_t status = i2cMasterTransmitTimeout(&I2C_DRIVER, (address >> 1), data, length, 0, 0, MS2ST(timeout));
+    msg_t status = i2cMasterTransmitTimeout(&I2C_DRIVER, (i2c_address >> 1), data, length, 0, 0, MS2ST(timeout));
-
-#if I2C_USE_MUTUAL_EXCLUSION
-    i2cReleaseBus(&I2C_DRIVER);
-#endif
-
     return chibios_to_qmk(&status);
 }
 
 i2c_status_t i2c_receive(uint8_t address, uint8_t* data, uint16_t length, uint16_t timeout) {
-#if I2C_USE_MUTUAL_EXCLUSION
+    i2c_address = address;
-    i2cAcquireBus(&I2C_DRIVER);
-#endif
-
     i2cStart(&I2C_DRIVER, &i2cconfig);
-    msg_t status = i2cMasterReceiveTimeout(&I2C_DRIVER, (address >> 1), data, length, MS2ST(timeout));
+    msg_t status = i2cMasterReceiveTimeout(&I2C_DRIVER, (i2c_address >> 1), data, length, MS2ST(timeout));
-
-#if I2C_USE_MUTUAL_EXCLUSION
-    i2cReleaseBus(&I2C_DRIVER);
-#endif
-
     return chibios_to_qmk(&status);
 }
 
 i2c_status_t i2c_writeReg(uint8_t devaddr, uint8_t regaddr, const uint8_t* data, uint16_t length, uint16_t timeout) {
-#if I2C_USE_MUTUAL_EXCLUSION
+    i2c_address = devaddr;
-    i2cAcquireBus(&I2C_DRIVER);
-#endif
-
     i2cStart(&I2C_DRIVER, &i2cconfig);
 
     uint8_t complete_packet[length + 1];
@@ -120,34 +100,15 @@ i2c_status_t i2c_writeReg(uint8_t devaddr, uint8_t regaddr, const uint8_t* data,
     }
     complete_packet[0] = regaddr;
 
-    msg_t status = i2cMasterTransmitTimeout(&I2C_DRIVER, (devaddr >> 1), complete_packet, length + 1, 0, 0, MS2ST(timeout));
+    msg_t status = i2cMasterTransmitTimeout(&I2C_DRIVER, (i2c_address >> 1), complete_packet, length + 1, 0, 0, MS2ST(timeout));
-
-#if I2C_USE_MUTUAL_EXCLUSION
-    i2cReleaseBus(&I2C_DRIVER);
-#endif
-
     return chibios_to_qmk(&status);
 }
 
 i2c_status_t i2c_readReg(uint8_t devaddr, uint8_t regaddr, uint8_t* data, uint16_t length, uint16_t timeout) {
-#if I2C_USE_MUTUAL_EXCLUSION
+    i2c_address = devaddr;
-    i2cAcquireBus(&I2C_DRIVER);
-#endif
-
     i2cStart(&I2C_DRIVER, &i2cconfig);
-    msg_t status = i2cMasterTransmitTimeout(&I2C_DRIVER, (devaddr >> 1), &regaddr, 1, data, length, MS2ST(timeout));
+    msg_t status = i2cMasterTransmitTimeout(&I2C_DRIVER, (i2c_address >> 1), &regaddr, 1, data, length, MS2ST(timeout));
-
-#if I2C_USE_MUTUAL_EXCLUSION
-    i2cReleaseBus(&I2C_DRIVER);
-#endif
-
     return chibios_to_qmk(&status);
 }
 
-void i2c_stop(void) {
+void i2c_stop(void) { i2cStop(&I2C_DRIVER); }
-    i2cStop(&I2C_DRIVER);
-
-#if I2C_USE_MUTUAL_EXCLUSION
-    i2cReleaseBus(&I2C_DRIVER);
-#endif
-}

lib/python/qmk/cli/cformat.py

@@ -1,16 +1,14 @@
 """Format C code according to QMK's style.
 """
-import os
 import subprocess
 from shutil import which
 
 from milc import cli
+import qmk.path
 
 
-@cli.argument('files', nargs='*', arg_only=True, help='Filename(s) to format.')
+def cformat_run(files, all_files):
-@cli.subcommand("Format C code according to QMK's style.")
+    """Spawn clang-format subprocess with proper arguments
-def cformat(cli):
-    """Format C code according to QMK's style.
     """
     # Determine which version of clang-format to use
     clang_format = ['clang-format', '-i']
@@ -19,27 +17,48 @@ def cformat(cli):
         if which(binary):
             clang_format[0] = binary
             break
-
-    # Find the list of files to format
-    if cli.args.files:
-        cli.args.files = [os.path.join(os.environ['ORIG_CWD'], file) for file in cli.args.files]
-    else:
-        ignores = ['tmk_core/protocol/usb_hid', 'quantum/template']
-        for dir in ['drivers', 'quantum', 'tests', 'tmk_core']:
-            for dirpath, dirnames, filenames in os.walk(dir):
-                if any(i in dirpath for i in ignores):
-                    dirnames.clear()
-                    continue
-
-                for name in filenames:
-                    if name.endswith(('.c', '.h', '.cpp')):
-                        cli.args.files.append(os.path.join(dirpath, name))
-
-    # Run clang-format on the files we've found
     try:
-        subprocess.run(clang_format + cli.args.files, check=True)
+        if not files:
+            cli.log.warn('No changes detected. Use "qmk cformat -a" to format all files')
+            return False
+        if files and all_files:
+            cli.log.warning('Filenames passed with -a, only formatting: %s', ','.join(cli.args.files))
+        # 3.6+: Can remove the str casting, python will cast implicitly
+        subprocess.run(clang_format + [str(file) for file in files], check=True)
         cli.log.info('Successfully formatted the C code.')
 
     except subprocess.CalledProcessError:
         cli.log.error('Error formatting C code!')
         return False
+
+
+@cli.argument('-a', '--all-files', arg_only=True, action='store_true', help='Format all core files.')
+@cli.argument('-b', '--base-branch', default='origin/master', help='Branch to compare to diffs to.')
+@cli.argument('files', nargs='*', arg_only=True, help='Filename(s) to format.')
+@cli.subcommand("Format C code according to QMK's style.")
+def cformat(cli):
+    """Format C code according to QMK's style.
+    """
+    # Empty array for files
+    files = []
+    # Core directories for formatting
+    core_dirs = ['drivers', 'quantum', 'tests', 'tmk_core']
+    ignores = ['tmk_core/protocol/usb_hid', 'quantum/template']
+    # Find the list of files to format
+    if cli.args.files:
+        files.extend(qmk.path.normpath(file) for file in cli.args.files)
+    # If -a is specified
+    elif cli.args.all_files:
+        all_files = qmk.path.c_source_files(core_dirs)
+        # The following statement checks each file to see if the file path is in the ignored directories.
+        files.extend(file for file in all_files if not any(i in str(file) for i in ignores))
+    # No files specified & no -a flag
+    else:
+        base_args = ['git', 'diff', '--name-only', cli.args.base_branch]
+        out = subprocess.run(base_args + core_dirs, check=True, stdout=subprocess.PIPE)
+        changed_files = filter(None, out.stdout.decode('UTF-8').split('\n'))
+        filtered_files = [qmk.path.normpath(file) for file in changed_files if not any(i in file for i in ignores)]
+        files.extend(file for file in filtered_files if file.exists() and file.suffix in ['.c', '.h', '.cpp'])
+
+    # Run clang-format on the files we've found
+    cformat_run(files, cli.args.all_files)

lib/python/qmk/path.py

@@ -68,3 +68,17 @@ def normpath(path):
         return Path(path)
 
     return Path(os.environ['ORIG_CWD']) / path
+
+
+def c_source_files(dir_names):
+    """Returns a list of all *.c, *.h, and *.cpp files for a given list of directories
+
+    Args:
+
+        dir_names
+            List of directories, relative pathing starts at qmk's cwd
+    """
+    files = []
+    for dir in dir_names:
+        files.extend(file for file in Path(dir).glob('**/*') if file.suffix in ['.c', '.h', '.cpp'])
+    return files

lib/python/qmk/tests/test_cli_commands.py

@@ -7,7 +7,8 @@ def check_subcommand(command, *args):
 
 
 def test_cformat():
-    assert check_subcommand('cformat', 'tmk_core/common/keyboard.c').returncode == 0
+    result = check_subcommand('cformat', 'quantum/matrix.c')
+    assert result.returncode == 0
 
 
 def test_compile():

quantum/process_keycode/process_unicode_common.c

@@ -178,6 +178,55 @@ void send_unicode_hex_string(const char *str) {
     }
 }
 
+// Borrowed from https://nullprogram.com/blog/2017/10/06/
+const char *decode_utf8(const char *str, int32_t *code_point) {
+    const char *next;
+
+    if (str[0] < 0x80) {  // U+0000-007F
+        *code_point = str[0];
+        next        = str + 1;
+    } else if ((str[0] & 0xE0) == 0xC0) {  // U+0080-07FF
+        *code_point = ((int32_t)(str[0] & 0x1F) << 6) | ((int32_t)(str[1] & 0x3F) << 0);
+        next        = str + 2;
+    } else if ((str[0] & 0xF0) == 0xE0) {  // U+0800-FFFF
+        *code_point = ((int32_t)(str[0] & 0x0F) << 12) | ((int32_t)(str[1] & 0x3F) << 6) | ((int32_t)(str[2] & 0x3F) << 0);
+        next        = str + 3;
+    } else if ((str[0] & 0xF8) == 0xF0 && (str[0] <= 0xF4)) {  // U+10000-10FFFF
+        // Skip for now - register_hex() only takes a uint16
+        //*code_point = ((int32_t)(str[0] & 0x07) << 18) | ((int32_t)(str[1] & 0x3F) << 12) | ((int32_t)(str[2] & 0x3F) << 6) | ((int32_t)(str[3] & 0x3F) << 0);
+        *code_point = -1;
+        next        = str + 4;
+    } else {
+        *code_point = -1;
+        next        = str + 1;
+    }
+
+    // part of a UTF-16 surrogate pair - invalid
+    if (*code_point >= 0xD800 && *code_point <= 0xDFFF) {
+        *code_point = -1;
+    }
+
+    return next;
+}
+
+void send_unicode_string(const char *str) {
+    if (!str) {
+        return;
+    }
+
+    int32_t code_point = 0;
+
+    while (*str) {
+        str = decode_utf8(str, &code_point);
+
+        if (code_point >= 0) {
+            unicode_input_start();
+            register_hex(code_point);
+            unicode_input_finish();
+        }
+    }
+}
+
 bool process_unicode_common(uint16_t keycode, keyrecord_t *record) {
     if (record->event.pressed) {
         switch (keycode) {

quantum/process_keycode/process_unicode_common.h

@@ -80,6 +80,7 @@ void unicode_input_cancel(void);
 
 void register_hex(uint16_t hex);
 void send_unicode_hex_string(const char *str);
+void send_unicode_string(const char *str);
 
 bool process_unicode_common(uint16_t keycode, keyrecord_t *record);
 

quantum/send_string_keycodes.h

@@ -137,6 +137,24 @@
 #define X_BRMU X_PAUSE
 #define X_BRMD X_SCROLLLOCK
 
+/* Mouse Keys */
+#define X_MS_U X_MS_UP
+#define X_MS_D X_MS_DOWN
+#define X_MS_L X_MS_LEFT
+#define X_MS_R X_MS_RIGHT
+#define X_BTN1 X_MS_BTN1
+#define X_BTN2 X_MS_BTN2
+#define X_BTN3 X_MS_BTN3
+#define X_BTN4 X_MS_BTN4
+#define X_BTN5 X_MS_BTN5
+#define X_WH_U X_MS_WH_UP
+#define X_WH_D X_MS_WH_DOWN
+#define X_WH_L X_MS_WH_LEFT
+#define X_WH_R X_MS_WH_RIGHT
+#define X_ACL0 X_MS_ACCEL0
+#define X_ACL1 X_MS_ACCEL1
+#define X_ACL2 X_MS_ACCEL2
+
 /* Keyboard/Keypad Page (0x07) */
 #define X_A                  04
 #define X_B                  05
@@ -341,6 +359,24 @@
 #define X_BRIGHTNESS_UP      bd
 #define X_BRIGHTNESS_DOWN    be
 
+/* Mouse Buttons (unallocated range in HID spec) */
+#define X_MS_UP              f0
+#define X_MS_DOWN            f1
+#define X_MS_LEFT            f2
+#define X_MS_RIGHT           f3
+#define X_MS_BTN1            f4
+#define X_MS_BTN2            f5
+#define X_MS_BTN3            f6
+#define X_MS_BTN4            f7
+#define X_MS_BTN5            f8
+#define X_MS_WH_UP           f9
+#define X_MS_WH_DOWN         fa
+#define X_MS_WH_LEFT         fb
+#define X_MS_WH_RIGHT        fc
+#define X_MS_ACCEL0          fd
+#define X_MS_ACCEL1          fe
+#define X_MS_ACCEL2          ff
+
 // Send string macros
 #define STRINGIZE(z) #z
 #define ADD_SLASH_X(y) STRINGIZE(\x##y)