persistentcode.c 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669
  1. /*
  2. * This file is part of the MicroPython project, http://micropython.org/
  3. *
  4. * The MIT License (MIT)
  5. *
  6. * Copyright (c) 2013-2020 Damien P. George
  7. *
  8. * Permission is hereby granted, free of charge, to any person obtaining a copy
  9. * of this software and associated documentation files (the "Software"), to deal
  10. * in the Software without restriction, including without limitation the rights
  11. * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  12. * copies of the Software, and to permit persons to whom the Software is
  13. * furnished to do so, subject to the following conditions:
  14. *
  15. * The above copyright notice and this permission notice shall be included in
  16. * all copies or substantial portions of the Software.
  17. *
  18. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19. * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20. * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  21. * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  22. * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  23. * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  24. * THE SOFTWARE.
  25. */
  26. #include <stdint.h>
  27. #include <stdio.h>
  28. #include <string.h>
  29. #include <assert.h>
  30. #include "py/reader.h"
  31. #include "py/nativeglue.h"
  32. #include "py/persistentcode.h"
  33. #include "py/bc0.h"
  34. #include "py/objstr.h"
  35. #include "py/mpthread.h"
  36. #if MICROPY_PERSISTENT_CODE_LOAD || MICROPY_PERSISTENT_CODE_SAVE
  37. #include "py/smallint.h"
  38. // makeqstrdata.py has a fixed list of qstrs at the start that we can assume
  39. // are available with know indices on all MicroPython implementations, and
  40. // avoid needing to duplicate the string data in the .mpy file. This is the
  41. // last one in that list (anything with a qstr less than or equal to this is
  42. // assumed to be in the list).
  43. #define QSTR_LAST_STATIC MP_QSTR_zip
  44. #if MICROPY_DYNAMIC_COMPILER
  45. #define MPY_FEATURE_ARCH_DYNAMIC mp_dynamic_compiler.native_arch
  46. #else
  47. #define MPY_FEATURE_ARCH_DYNAMIC MPY_FEATURE_ARCH
  48. #endif
  49. typedef struct _bytecode_prelude_t {
  50. uint n_state;
  51. uint n_exc_stack;
  52. uint scope_flags;
  53. uint n_pos_args;
  54. uint n_kwonly_args;
  55. uint n_def_pos_args;
  56. uint code_info_size;
  57. } bytecode_prelude_t;
  58. #endif // MICROPY_PERSISTENT_CODE_LOAD || MICROPY_PERSISTENT_CODE_SAVE
  59. #if MICROPY_PERSISTENT_CODE_LOAD
  60. #include "py/parsenum.h"
  61. static int read_byte(mp_reader_t *reader);
  62. static size_t read_uint(mp_reader_t *reader);
  63. #if MICROPY_EMIT_MACHINE_CODE
  64. typedef struct _reloc_info_t {
  65. mp_reader_t *reader;
  66. mp_module_context_t *context;
  67. uint8_t *rodata;
  68. uint8_t *bss;
  69. } reloc_info_t;
  70. void mp_native_relocate(void *ri_in, uint8_t *text, uintptr_t reloc_text) {
  71. // Relocate native code
  72. reloc_info_t *ri = ri_in;
  73. uint8_t op;
  74. uintptr_t *addr_to_adjust = NULL;
  75. while ((op = read_byte(ri->reader)) != 0xff) {
  76. if (op & 1) {
  77. // Point to new location to make adjustments
  78. size_t addr = read_uint(ri->reader);
  79. if ((addr & 1) == 0) {
  80. // Point to somewhere in text
  81. addr_to_adjust = &((uintptr_t *)text)[addr >> 1];
  82. } else {
  83. // Point to somewhere in rodata
  84. addr_to_adjust = &((uintptr_t *)ri->rodata)[addr >> 1];
  85. }
  86. }
  87. op >>= 1;
  88. uintptr_t dest;
  89. size_t n = 1;
  90. if (op <= 5) {
  91. if (op & 1) {
  92. // Read in number of adjustments to make
  93. n = read_uint(ri->reader);
  94. }
  95. op >>= 1;
  96. if (op == 0) {
  97. // Destination is text
  98. dest = reloc_text;
  99. } else if (op == 1) {
  100. // Destination is rodata
  101. dest = (uintptr_t)ri->rodata;
  102. } else {
  103. // Destination is bss
  104. dest = (uintptr_t)ri->bss;
  105. }
  106. } else if (op == 6) {
  107. // Destination is qstr_table
  108. dest = (uintptr_t)ri->context->constants.qstr_table;
  109. } else if (op == 7) {
  110. // Destination is obj_table
  111. dest = (uintptr_t)ri->context->constants.obj_table;
  112. } else if (op == 8) {
  113. // Destination is mp_fun_table itself
  114. dest = (uintptr_t)&mp_fun_table;
  115. } else {
  116. // Destination is an entry in mp_fun_table
  117. dest = ((uintptr_t *)&mp_fun_table)[op - 9];
  118. }
  119. while (n--) {
  120. *addr_to_adjust++ += dest;
  121. }
  122. }
  123. }
  124. #endif
  125. static int read_byte(mp_reader_t *reader) {
  126. return reader->readbyte(reader->data);
  127. }
  128. static void read_bytes(mp_reader_t *reader, byte *buf, size_t len) {
  129. while (len-- > 0) {
  130. *buf++ = reader->readbyte(reader->data);
  131. }
  132. }
  133. static size_t read_uint(mp_reader_t *reader) {
  134. size_t unum = 0;
  135. for (;;) {
  136. byte b = reader->readbyte(reader->data);
  137. unum = (unum << 7) | (b & 0x7f);
  138. if ((b & 0x80) == 0) {
  139. break;
  140. }
  141. }
  142. return unum;
  143. }
  144. static qstr load_qstr(mp_reader_t *reader) {
  145. size_t len = read_uint(reader);
  146. if (len & 1) {
  147. // static qstr
  148. return len >> 1;
  149. }
  150. len >>= 1;
  151. char *str = m_new(char, len);
  152. read_bytes(reader, (byte *)str, len);
  153. read_byte(reader); // read and discard null terminator
  154. qstr qst = qstr_from_strn(str, len);
  155. m_del(char, str, len);
  156. return qst;
  157. }
  158. static mp_obj_t load_obj(mp_reader_t *reader) {
  159. byte obj_type = read_byte(reader);
  160. #if MICROPY_EMIT_MACHINE_CODE
  161. if (obj_type == MP_PERSISTENT_OBJ_FUN_TABLE) {
  162. return MP_OBJ_FROM_PTR(&mp_fun_table);
  163. } else
  164. #endif
  165. if (obj_type == MP_PERSISTENT_OBJ_NONE) {
  166. return mp_const_none;
  167. } else if (obj_type == MP_PERSISTENT_OBJ_FALSE) {
  168. return mp_const_false;
  169. } else if (obj_type == MP_PERSISTENT_OBJ_TRUE) {
  170. return mp_const_true;
  171. } else if (obj_type == MP_PERSISTENT_OBJ_ELLIPSIS) {
  172. return MP_OBJ_FROM_PTR(&mp_const_ellipsis_obj);
  173. } else {
  174. size_t len = read_uint(reader);
  175. if (len == 0 && obj_type == MP_PERSISTENT_OBJ_BYTES) {
  176. read_byte(reader); // skip null terminator
  177. return mp_const_empty_bytes;
  178. } else if (obj_type == MP_PERSISTENT_OBJ_TUPLE) {
  179. mp_obj_tuple_t *tuple = MP_OBJ_TO_PTR(mp_obj_new_tuple(len, NULL));
  180. for (size_t i = 0; i < len; ++i) {
  181. tuple->items[i] = load_obj(reader);
  182. }
  183. return MP_OBJ_FROM_PTR(tuple);
  184. }
  185. vstr_t vstr;
  186. vstr_init_len(&vstr, len);
  187. read_bytes(reader, (byte *)vstr.buf, len);
  188. if (obj_type == MP_PERSISTENT_OBJ_STR || obj_type == MP_PERSISTENT_OBJ_BYTES) {
  189. read_byte(reader); // skip null terminator
  190. if (obj_type == MP_PERSISTENT_OBJ_STR) {
  191. return mp_obj_new_str_from_utf8_vstr(&vstr);
  192. } else {
  193. return mp_obj_new_bytes_from_vstr(&vstr);
  194. }
  195. } else if (obj_type == MP_PERSISTENT_OBJ_INT) {
  196. return mp_parse_num_integer(vstr.buf, vstr.len, 10, NULL);
  197. } else {
  198. assert(obj_type == MP_PERSISTENT_OBJ_FLOAT || obj_type == MP_PERSISTENT_OBJ_COMPLEX);
  199. return mp_parse_num_float(vstr.buf, vstr.len, obj_type == MP_PERSISTENT_OBJ_COMPLEX, NULL);
  200. }
  201. }
  202. }
  203. static mp_raw_code_t *load_raw_code(mp_reader_t *reader, mp_module_context_t *context) {
  204. // Load function kind and data length
  205. size_t kind_len = read_uint(reader);
  206. int kind = (kind_len & 3) + MP_CODE_BYTECODE;
  207. bool has_children = !!(kind_len & 4);
  208. size_t fun_data_len = kind_len >> 3;
  209. #if !MICROPY_EMIT_MACHINE_CODE
  210. if (kind != MP_CODE_BYTECODE) {
  211. mp_raise_ValueError(MP_ERROR_TEXT("incompatible .mpy file"));
  212. }
  213. #endif
  214. uint8_t *fun_data = NULL;
  215. #if MICROPY_EMIT_MACHINE_CODE
  216. size_t prelude_offset = 0;
  217. mp_uint_t native_scope_flags = 0;
  218. mp_uint_t native_n_pos_args = 0;
  219. mp_uint_t native_type_sig = 0;
  220. #endif
  221. if (kind == MP_CODE_BYTECODE) {
  222. // Allocate memory for the bytecode
  223. fun_data = m_new(uint8_t, fun_data_len);
  224. // Load bytecode
  225. read_bytes(reader, fun_data, fun_data_len);
  226. #if MICROPY_EMIT_MACHINE_CODE
  227. } else {
  228. // Allocate memory for native data and load it
  229. size_t fun_alloc;
  230. MP_PLAT_ALLOC_EXEC(fun_data_len, (void **)&fun_data, &fun_alloc);
  231. read_bytes(reader, fun_data, fun_data_len);
  232. if (kind == MP_CODE_NATIVE_PY) {
  233. // Read prelude offset within fun_data, and extract scope flags.
  234. prelude_offset = read_uint(reader);
  235. const byte *ip = fun_data + prelude_offset;
  236. MP_BC_PRELUDE_SIG_DECODE(ip);
  237. native_scope_flags = scope_flags;
  238. } else {
  239. // Load basic scope info for viper and asm.
  240. native_scope_flags = read_uint(reader);
  241. if (kind == MP_CODE_NATIVE_ASM) {
  242. native_n_pos_args = read_uint(reader);
  243. native_type_sig = read_uint(reader);
  244. }
  245. }
  246. #endif
  247. }
  248. size_t n_children = 0;
  249. mp_raw_code_t **children = NULL;
  250. #if MICROPY_EMIT_MACHINE_CODE
  251. // Load optional BSS/rodata for viper.
  252. uint8_t *rodata = NULL;
  253. uint8_t *bss = NULL;
  254. if (kind == MP_CODE_NATIVE_VIPER) {
  255. size_t rodata_size = 0;
  256. if (native_scope_flags & MP_SCOPE_FLAG_VIPERRODATA) {
  257. rodata_size = read_uint(reader);
  258. }
  259. size_t bss_size = 0;
  260. if (native_scope_flags & MP_SCOPE_FLAG_VIPERBSS) {
  261. bss_size = read_uint(reader);
  262. }
  263. if (rodata_size + bss_size != 0) {
  264. bss_size = (uintptr_t)MP_ALIGN(bss_size, sizeof(uintptr_t));
  265. uint8_t *data = m_new0(uint8_t, bss_size + rodata_size);
  266. bss = data;
  267. rodata = bss + bss_size;
  268. if (native_scope_flags & MP_SCOPE_FLAG_VIPERRODATA) {
  269. read_bytes(reader, rodata, rodata_size);
  270. }
  271. // Viper code with BSS/rodata should not have any children.
  272. // Reuse the children pointer to reference the BSS/rodata
  273. // memory so that it is not reclaimed by the GC.
  274. assert(!has_children);
  275. children = (void *)data;
  276. }
  277. }
  278. #endif
  279. // Load children if any.
  280. if (has_children) {
  281. n_children = read_uint(reader);
  282. children = m_new(mp_raw_code_t *, n_children + (kind == MP_CODE_NATIVE_PY));
  283. for (size_t i = 0; i < n_children; ++i) {
  284. children[i] = load_raw_code(reader, context);
  285. }
  286. }
  287. // Create raw_code and return it
  288. mp_raw_code_t *rc = mp_emit_glue_new_raw_code();
  289. if (kind == MP_CODE_BYTECODE) {
  290. const byte *ip = fun_data;
  291. MP_BC_PRELUDE_SIG_DECODE(ip);
  292. // Assign bytecode to raw code object
  293. mp_emit_glue_assign_bytecode(rc, fun_data,
  294. children,
  295. #if MICROPY_PERSISTENT_CODE_SAVE
  296. fun_data_len,
  297. n_children,
  298. #endif
  299. scope_flags);
  300. #if MICROPY_EMIT_MACHINE_CODE
  301. } else {
  302. const uint8_t *prelude_ptr;
  303. #if MICROPY_EMIT_NATIVE_PRELUDE_SEPARATE_FROM_MACHINE_CODE
  304. if (kind == MP_CODE_NATIVE_PY) {
  305. // Executable code cannot be accessed byte-wise on this architecture, so copy
  306. // the prelude to a separate memory region that is byte-wise readable.
  307. void *buf = fun_data + prelude_offset;
  308. size_t n = fun_data_len - prelude_offset;
  309. prelude_ptr = memcpy(m_new(uint8_t, n), buf, n);
  310. }
  311. #endif
  312. // Relocate and commit code to executable address space
  313. reloc_info_t ri = {reader, context, rodata, bss};
  314. #if defined(MP_PLAT_COMMIT_EXEC)
  315. void *opt_ri = (native_scope_flags & MP_SCOPE_FLAG_VIPERRELOC) ? &ri : NULL;
  316. fun_data = MP_PLAT_COMMIT_EXEC(fun_data, fun_data_len, opt_ri);
  317. #else
  318. if (native_scope_flags & MP_SCOPE_FLAG_VIPERRELOC) {
  319. #if MICROPY_PERSISTENT_CODE_TRACK_RELOC_CODE
  320. // If native code needs relocations then it's not guaranteed that a pointer to
  321. // the head of `buf` (containing the machine code) will be retained for the GC
  322. // to trace. This is because native functions can start inside `buf` and so
  323. // it's possible that the only GC-reachable pointers are pointers inside `buf`.
  324. // So put this `buf` on a list of reachable root pointers.
  325. if (MP_STATE_PORT(track_reloc_code_list) == MP_OBJ_NULL) {
  326. MP_STATE_PORT(track_reloc_code_list) = mp_obj_new_list(0, NULL);
  327. }
  328. mp_obj_list_append(MP_STATE_PORT(track_reloc_code_list), MP_OBJ_FROM_PTR(fun_data));
  329. #endif
  330. // Do the relocations.
  331. mp_native_relocate(&ri, fun_data, (uintptr_t)fun_data);
  332. }
  333. #endif
  334. if (kind == MP_CODE_NATIVE_PY) {
  335. #if !MICROPY_EMIT_NATIVE_PRELUDE_SEPARATE_FROM_MACHINE_CODE
  336. prelude_ptr = fun_data + prelude_offset;
  337. #endif
  338. if (n_children == 0) {
  339. children = (void *)prelude_ptr;
  340. } else {
  341. children[n_children] = (void *)prelude_ptr;
  342. }
  343. }
  344. // Assign native code to raw code object
  345. mp_emit_glue_assign_native(rc, kind,
  346. fun_data, fun_data_len,
  347. children,
  348. #if MICROPY_PERSISTENT_CODE_SAVE
  349. n_children,
  350. prelude_offset,
  351. #endif
  352. native_scope_flags, native_n_pos_args, native_type_sig
  353. );
  354. #endif
  355. }
  356. return rc;
  357. }
  358. void mp_raw_code_load(mp_reader_t *reader, mp_compiled_module_t *cm) {
  359. // Set exception handler to close the reader if an exception is raised.
  360. MP_DEFINE_NLR_JUMP_CALLBACK_FUNCTION_1(ctx, reader->close, reader->data);
  361. nlr_push_jump_callback(&ctx.callback, mp_call_function_1_from_nlr_jump_callback);
  362. byte header[4];
  363. read_bytes(reader, header, sizeof(header));
  364. byte arch = MPY_FEATURE_DECODE_ARCH(header[2]);
  365. if (header[0] != 'M'
  366. || header[1] != MPY_VERSION
  367. || (arch != MP_NATIVE_ARCH_NONE && MPY_FEATURE_DECODE_SUB_VERSION(header[2]) != MPY_SUB_VERSION)
  368. || header[3] > MP_SMALL_INT_BITS) {
  369. mp_raise_ValueError(MP_ERROR_TEXT("incompatible .mpy file"));
  370. }
  371. if (MPY_FEATURE_DECODE_ARCH(header[2]) != MP_NATIVE_ARCH_NONE) {
  372. if (!MPY_FEATURE_ARCH_TEST(arch)) {
  373. if (MPY_FEATURE_ARCH_TEST(MP_NATIVE_ARCH_NONE)) {
  374. // On supported ports this can be resolved by enabling feature, eg
  375. // mpconfigboard.h: MICROPY_EMIT_THUMB (1)
  376. mp_raise_ValueError(MP_ERROR_TEXT("native code in .mpy unsupported"));
  377. } else {
  378. mp_raise_ValueError(MP_ERROR_TEXT("incompatible .mpy arch"));
  379. }
  380. }
  381. }
  382. size_t n_qstr = read_uint(reader);
  383. size_t n_obj = read_uint(reader);
  384. mp_module_context_alloc_tables(cm->context, n_qstr, n_obj);
  385. // Load qstrs.
  386. for (size_t i = 0; i < n_qstr; ++i) {
  387. cm->context->constants.qstr_table[i] = load_qstr(reader);
  388. }
  389. // Load constant objects.
  390. for (size_t i = 0; i < n_obj; ++i) {
  391. cm->context->constants.obj_table[i] = load_obj(reader);
  392. }
  393. // Load top-level module.
  394. cm->rc = load_raw_code(reader, cm->context);
  395. #if MICROPY_PERSISTENT_CODE_SAVE
  396. cm->has_native = MPY_FEATURE_DECODE_ARCH(header[2]) != MP_NATIVE_ARCH_NONE;
  397. cm->n_qstr = n_qstr;
  398. cm->n_obj = n_obj;
  399. #endif
  400. // Deregister exception handler and close the reader.
  401. nlr_pop_jump_callback(true);
  402. }
  403. void mp_raw_code_load_mem(const byte *buf, size_t len, mp_compiled_module_t *context) {
  404. mp_reader_t reader;
  405. mp_reader_new_mem(&reader, buf, len, 0);
  406. mp_raw_code_load(&reader, context);
  407. }
  408. #if MICROPY_HAS_FILE_READER
  409. void mp_raw_code_load_file(qstr filename, mp_compiled_module_t *context) {
  410. mp_reader_t reader;
  411. mp_reader_new_file(&reader, filename);
  412. mp_raw_code_load(&reader, context);
  413. }
  414. #endif // MICROPY_HAS_FILE_READER
  415. #endif // MICROPY_PERSISTENT_CODE_LOAD
  416. #if MICROPY_PERSISTENT_CODE_SAVE
  417. #include "py/objstr.h"
  418. static void mp_print_bytes(mp_print_t *print, const byte *data, size_t len) {
  419. print->print_strn(print->data, (const char *)data, len);
  420. }
  421. #define BYTES_FOR_INT ((MP_BYTES_PER_OBJ_WORD * 8 + 6) / 7)
  422. static void mp_print_uint(mp_print_t *print, size_t n) {
  423. byte buf[BYTES_FOR_INT];
  424. byte *p = buf + sizeof(buf);
  425. *--p = n & 0x7f;
  426. n >>= 7;
  427. for (; n != 0; n >>= 7) {
  428. *--p = 0x80 | (n & 0x7f);
  429. }
  430. print->print_strn(print->data, (char *)p, buf + sizeof(buf) - p);
  431. }
  432. static void save_qstr(mp_print_t *print, qstr qst) {
  433. if (qst <= QSTR_LAST_STATIC) {
  434. // encode static qstr
  435. mp_print_uint(print, qst << 1 | 1);
  436. return;
  437. }
  438. size_t len;
  439. const byte *str = qstr_data(qst, &len);
  440. mp_print_uint(print, len << 1);
  441. mp_print_bytes(print, str, len + 1); // +1 to store null terminator
  442. }
  443. static void save_obj(mp_print_t *print, mp_obj_t o) {
  444. #if MICROPY_EMIT_MACHINE_CODE
  445. if (o == MP_OBJ_FROM_PTR(&mp_fun_table)) {
  446. byte obj_type = MP_PERSISTENT_OBJ_FUN_TABLE;
  447. mp_print_bytes(print, &obj_type, 1);
  448. } else
  449. #endif
  450. if (mp_obj_is_str_or_bytes(o)) {
  451. byte obj_type;
  452. if (mp_obj_is_str(o)) {
  453. obj_type = MP_PERSISTENT_OBJ_STR;
  454. } else {
  455. obj_type = MP_PERSISTENT_OBJ_BYTES;
  456. }
  457. size_t len;
  458. const char *str = mp_obj_str_get_data(o, &len);
  459. mp_print_bytes(print, &obj_type, 1);
  460. mp_print_uint(print, len);
  461. mp_print_bytes(print, (const byte *)str, len + 1); // +1 to store null terminator
  462. } else if (o == mp_const_none) {
  463. byte obj_type = MP_PERSISTENT_OBJ_NONE;
  464. mp_print_bytes(print, &obj_type, 1);
  465. } else if (o == mp_const_false) {
  466. byte obj_type = MP_PERSISTENT_OBJ_FALSE;
  467. mp_print_bytes(print, &obj_type, 1);
  468. } else if (o == mp_const_true) {
  469. byte obj_type = MP_PERSISTENT_OBJ_TRUE;
  470. mp_print_bytes(print, &obj_type, 1);
  471. } else if (MP_OBJ_TO_PTR(o) == &mp_const_ellipsis_obj) {
  472. byte obj_type = MP_PERSISTENT_OBJ_ELLIPSIS;
  473. mp_print_bytes(print, &obj_type, 1);
  474. } else if (mp_obj_is_type(o, &mp_type_tuple)) {
  475. size_t len;
  476. mp_obj_t *items;
  477. mp_obj_tuple_get(o, &len, &items);
  478. byte obj_type = MP_PERSISTENT_OBJ_TUPLE;
  479. mp_print_bytes(print, &obj_type, 1);
  480. mp_print_uint(print, len);
  481. for (size_t i = 0; i < len; ++i) {
  482. save_obj(print, items[i]);
  483. }
  484. } else {
  485. // we save numbers using a simplistic text representation
  486. // TODO could be improved
  487. byte obj_type;
  488. if (mp_obj_is_int(o)) {
  489. obj_type = MP_PERSISTENT_OBJ_INT;
  490. #if MICROPY_PY_BUILTINS_COMPLEX
  491. } else if (mp_obj_is_type(o, &mp_type_complex)) {
  492. obj_type = MP_PERSISTENT_OBJ_COMPLEX;
  493. #endif
  494. } else {
  495. assert(mp_obj_is_float(o));
  496. obj_type = MP_PERSISTENT_OBJ_FLOAT;
  497. }
  498. vstr_t vstr;
  499. mp_print_t pr;
  500. vstr_init_print(&vstr, 10, &pr);
  501. mp_obj_print_helper(&pr, o, PRINT_REPR);
  502. mp_print_bytes(print, &obj_type, 1);
  503. mp_print_uint(print, vstr.len);
  504. mp_print_bytes(print, (const byte *)vstr.buf, vstr.len);
  505. vstr_clear(&vstr);
  506. }
  507. }
  508. static void save_raw_code(mp_print_t *print, const mp_raw_code_t *rc) {
  509. // Save function kind and data length
  510. mp_print_uint(print, (rc->fun_data_len << 3) | ((rc->n_children != 0) << 2) | (rc->kind - MP_CODE_BYTECODE));
  511. // Save function code.
  512. mp_print_bytes(print, rc->fun_data, rc->fun_data_len);
  513. #if MICROPY_EMIT_MACHINE_CODE
  514. if (rc->kind == MP_CODE_NATIVE_PY) {
  515. // Save prelude size
  516. mp_print_uint(print, rc->prelude_offset);
  517. } else if (rc->kind == MP_CODE_NATIVE_VIPER || rc->kind == MP_CODE_NATIVE_ASM) {
  518. // Save basic scope info for viper and asm
  519. // Viper/asm functions don't support generator, variable args, or default keyword args
  520. // so (scope_flags & MP_SCOPE_FLAG_ALL_SIG) for these functions is always 0.
  521. mp_print_uint(print, 0);
  522. #if MICROPY_EMIT_INLINE_ASM
  523. if (rc->kind == MP_CODE_NATIVE_ASM) {
  524. mp_print_uint(print, rc->asm_n_pos_args);
  525. mp_print_uint(print, rc->asm_type_sig);
  526. }
  527. #endif
  528. }
  529. #endif
  530. if (rc->n_children) {
  531. mp_print_uint(print, rc->n_children);
  532. for (size_t i = 0; i < rc->n_children; ++i) {
  533. save_raw_code(print, rc->children[i]);
  534. }
  535. }
  536. }
  537. void mp_raw_code_save(mp_compiled_module_t *cm, mp_print_t *print) {
  538. // header contains:
  539. // byte 'M'
  540. // byte version
  541. // byte native arch (and sub-version if native)
  542. // byte number of bits in a small int
  543. byte header[4] = {
  544. 'M',
  545. MPY_VERSION,
  546. cm->has_native ? MPY_FEATURE_ENCODE_SUB_VERSION(MPY_SUB_VERSION) | MPY_FEATURE_ENCODE_ARCH(MPY_FEATURE_ARCH_DYNAMIC) : 0,
  547. #if MICROPY_DYNAMIC_COMPILER
  548. mp_dynamic_compiler.small_int_bits,
  549. #else
  550. MP_SMALL_INT_BITS,
  551. #endif
  552. };
  553. mp_print_bytes(print, header, sizeof(header));
  554. // Number of entries in constant table.
  555. mp_print_uint(print, cm->n_qstr);
  556. mp_print_uint(print, cm->n_obj);
  557. // Save qstrs.
  558. for (size_t i = 0; i < cm->n_qstr; ++i) {
  559. save_qstr(print, cm->context->constants.qstr_table[i]);
  560. }
  561. // Save constant objects.
  562. for (size_t i = 0; i < cm->n_obj; ++i) {
  563. save_obj(print, (mp_obj_t)cm->context->constants.obj_table[i]);
  564. }
  565. // Save outer raw code, which will save all its child raw codes.
  566. save_raw_code(print, cm->rc);
  567. }
  568. #if MICROPY_PERSISTENT_CODE_SAVE_FILE
  569. #include <unistd.h>
  570. #include <sys/stat.h>
  571. #include <fcntl.h>
  572. static void fd_print_strn(void *env, const char *str, size_t len) {
  573. int fd = (intptr_t)env;
  574. MP_THREAD_GIL_EXIT();
  575. ssize_t ret = write(fd, str, len);
  576. MP_THREAD_GIL_ENTER();
  577. (void)ret;
  578. }
  579. void mp_raw_code_save_file(mp_compiled_module_t *cm, qstr filename) {
  580. MP_THREAD_GIL_EXIT();
  581. int fd = open(qstr_str(filename), O_WRONLY | O_CREAT | O_TRUNC, 0644);
  582. MP_THREAD_GIL_ENTER();
  583. if (fd < 0) {
  584. mp_raise_OSError_with_filename(errno, qstr_str(filename));
  585. }
  586. mp_print_t fd_print = {(void *)(intptr_t)fd, fd_print_strn};
  587. mp_raw_code_save(cm, &fd_print);
  588. MP_THREAD_GIL_EXIT();
  589. close(fd);
  590. MP_THREAD_GIL_ENTER();
  591. }
  592. #endif // MICROPY_PERSISTENT_CODE_SAVE_FILE
  593. #endif // MICROPY_PERSISTENT_CODE_SAVE
  594. #if MICROPY_PERSISTENT_CODE_TRACK_RELOC_CODE
  595. // An mp_obj_list_t that tracks relocated native code to prevent the GC from reclaiming them.
  596. MP_REGISTER_ROOT_POINTER(mp_obj_t track_reloc_code_list);
  597. #endif