From 5e20a29b4fdc8a2d442d1093681b396dcb4b816b Mon Sep 17 00:00:00 2001 From: Robin Krahl Date: Tue, 7 Jan 2020 11:18:04 +0000 Subject: Add structopt dependency in version 0.3.7 This patch series replaces argparse with structopt in the argument handling code. As a first step, we need structopt as a dependency. Import subrepo structopt/:structopt at efbdda4753592e27bc430fb01f7b9650b2f3174d Import subrepo bitflags/:bitflags at 30668016aca6bd3b02c766e8347e0b4080d4c296 Import subrepo clap/:clap at 784524f7eb193e35f81082cc69454c8c21b948f7 Import subrepo heck/:heck at 093d56fbf001e1506e56dbfa38631d99b1066df1 Import subrepo proc-macro-error/:proc-macro-error at 6c4cfe79a622c5de8ae68557993542be46eacae2 Import subrepo proc-macro2/:proc-macro2 at d5d48eddca4566e5438e8a2cbed4a74e049544de Import subrepo quote/:quote at 727436c6c137b20f0f34dde5d8fda2679b9747ad Import subrepo rustversion/:rustversion at 0c5663313516263059ce9059ef81fc7a1cf655ca Import subrepo syn-mid/:syn-mid at 5d3d85414a9e6674e1857ec22a87b96e04a6851a Import subrepo syn/:syn at e87c27e87f6f4ef8919d0372bdb056d53ef0d8f3 Import subrepo textwrap/:textwrap at abcd618beae3f74841032aa5b53c1086b0a57ca2 Import subrepo unicode-segmentation/:unicode-segmentation at 637c9874c4fe0c205ff27787faf150a40295c6c3 Import subrepo unicode-width/:unicode-width at 3033826f8bf05e82724140a981d5941e48fce393 Import subrepo unicode-xid/:unicode-xid at 4baae9fffb156ba229665b972a9cd5991787ceb7 --- syn/.gitignore | 3 + syn/.travis.yml | 76 + syn/Cargo.toml | 72 + syn/LICENSE-APACHE | 201 + syn/LICENSE-MIT | 23 + syn/README.md | 291 ++ syn/appveyor.yml | 16 + syn/benches/file.rs | 30 + syn/benches/rust.rs | 158 + syn/build.rs | 63 + syn/codegen/Cargo.toml | 31 + syn/codegen/README.md | 12 + syn/codegen/src/debug.rs | 308 ++ syn/codegen/src/file.rs | 32 + syn/codegen/src/fold.rs | 284 ++ syn/codegen/src/full.rs | 20 + syn/codegen/src/gen.rs | 45 + syn/codegen/src/json.rs | 18 + syn/codegen/src/main.rs | 36 + syn/codegen/src/operand.rs | 38 + syn/codegen/src/parse.rs | 657 +++ syn/codegen/src/version.rs | 24 + syn/codegen/src/visit.rs | 265 + syn/codegen/src/visit_mut.rs | 262 + syn/dev/Cargo.toml | 22 + syn/dev/README.md | 6 + syn/dev/main.rs | 4 + syn/dev/parse.rs | 18 + syn/examples/README.md | 19 + syn/examples/dump-syntax/Cargo.toml | 17 + syn/examples/dump-syntax/README.md | 28 + syn/examples/dump-syntax/src/main.rs | 149 + syn/examples/heapsize/Cargo.toml | 2 + syn/examples/heapsize/README.md | 72 + syn/examples/heapsize/example/Cargo.toml | 9 + syn/examples/heapsize/example/src/main.rs | 28 + syn/examples/heapsize/heapsize/Cargo.toml | 9 + syn/examples/heapsize/heapsize/src/lib.rs | 64 + syn/examples/heapsize/heapsize_derive/Cargo.toml | 14 + syn/examples/heapsize/heapsize_derive/src/lib.rs | 96 + syn/examples/lazy-static/Cargo.toml | 2 + syn/examples/lazy-static/README.md | 42 + syn/examples/lazy-static/example/Cargo.toml | 10 + syn/examples/lazy-static/example/src/main.rs | 20 + syn/examples/lazy-static/lazy-static/Cargo.toml | 14 + syn/examples/lazy-static/lazy-static/src/lib.rs | 143 + syn/examples/trace-var/Cargo.toml | 2 + syn/examples/trace-var/README.md | 61 + syn/examples/trace-var/example/Cargo.toml | 9 + syn/examples/trace-var/example/src/main.rs | 15 + syn/examples/trace-var/trace-var/Cargo.toml | 14 + syn/examples/trace-var/trace-var/src/lib.rs | 180 + syn/json/Cargo.toml | 18 + syn/json/src/lib.rs | 214 + syn/src/attr.rs | 682 +++ syn/src/await.rs | 2 + syn/src/bigint.rs | 66 + syn/src/buffer.rs | 382 ++ syn/src/custom_keyword.rs | 252 + syn/src/custom_punctuation.rs | 309 ++ syn/src/data.rs | 456 ++ syn/src/derive.rs | 273 ++ syn/src/discouraged.rs | 195 + syn/src/error.rs | 357 ++ syn/src/export.rs | 35 + syn/src/expr.rs | 3236 +++++++++++++ syn/src/ext.rs | 135 + syn/src/file.rs | 113 + syn/src/gen/fold.rs | 3236 +++++++++++++ syn/src/gen/visit.rs | 3792 +++++++++++++++ syn/src/gen/visit_mut.rs | 3798 +++++++++++++++ syn/src/gen_helper.rs | 154 + syn/src/generics.rs | 1152 +++++ syn/src/group.rs | 280 ++ syn/src/ident.rs | 101 + syn/src/item.rs | 3104 ++++++++++++ syn/src/keyword.rs | 0 syn/src/lib.rs | 947 ++++ syn/src/lifetime.rs | 140 + syn/src/lit.rs | 1382 ++++++ syn/src/lookahead.rs | 168 + syn/src/mac.rs | 239 + syn/src/macros.rs | 191 + syn/src/op.rs | 231 + syn/src/parse.rs | 1222 +++++ syn/src/parse_macro_input.rs | 110 + syn/src/parse_quote.rs | 131 + syn/src/pat.rs | 903 ++++ syn/src/path.rs | 744 +++ syn/src/print.rs | 16 + syn/src/punctuated.rs | 918 ++++ syn/src/sealed.rs | 4 + syn/src/span.rs | 67 + syn/src/spanned.rs | 114 + syn/src/stmt.rs | 333 ++ syn/src/thread.rs | 41 + syn/src/token.rs | 956 ++++ syn/src/tt.rs | 108 + syn/src/ty.rs | 1178 +++++ syn/syn.json | 5616 +++++++++++++++++++++ syn/tests/common/eq.rs | 459 ++ syn/tests/common/mod.rs | 19 + syn/tests/common/parse.rs | 49 + syn/tests/debug/gen.rs | 5633 ++++++++++++++++++++++ syn/tests/debug/mod.rs | 110 + syn/tests/features/error.rs | 1 + syn/tests/features/mod.rs | 22 + syn/tests/macros/mod.rs | 76 + syn/tests/repo/mod.rs | 109 + syn/tests/repo/progress.rs | 37 + syn/tests/test_asyncness.rs | 39 + syn/tests/test_attribute.rs | 296 ++ syn/tests/test_derive_input.rs | 894 ++++ syn/tests/test_expr.rs | 37 + syn/tests/test_generics.rs | 285 ++ syn/tests/test_grouping.rs | 55 + syn/tests/test_ident.rs | 87 + syn/tests/test_iterators.rs | 51 + syn/tests/test_lit.rs | 184 + syn/tests/test_meta.rs | 341 ++ syn/tests/test_parse_buffer.rs | 53 + syn/tests/test_pat.rs | 20 + syn/tests/test_precedence.rs | 394 ++ syn/tests/test_receiver.rs | 109 + syn/tests/test_round_trip.rs | 151 + syn/tests/test_should_parse.rs | 47 + syn/tests/test_size.rs | 31 + syn/tests/test_token_trees.rs | 28 + syn/tests/test_visibility.rs | 99 + syn/tests/zzz_stable.rs | 33 + 130 files changed, 51884 insertions(+) create mode 100644 syn/.gitignore create mode 100644 syn/.travis.yml create mode 100644 syn/Cargo.toml create mode 100644 syn/LICENSE-APACHE create mode 100644 syn/LICENSE-MIT create mode 100644 syn/README.md create mode 100644 syn/appveyor.yml create mode 100644 syn/benches/file.rs create mode 100644 syn/benches/rust.rs create mode 100644 syn/build.rs create mode 100644 syn/codegen/Cargo.toml create mode 100644 syn/codegen/README.md create mode 100644 syn/codegen/src/debug.rs create mode 100644 syn/codegen/src/file.rs create mode 100644 syn/codegen/src/fold.rs create mode 100644 syn/codegen/src/full.rs create mode 100644 syn/codegen/src/gen.rs create mode 100644 syn/codegen/src/json.rs create mode 100644 syn/codegen/src/main.rs create mode 100644 syn/codegen/src/operand.rs create mode 100644 syn/codegen/src/parse.rs create mode 100644 syn/codegen/src/version.rs create mode 100644 syn/codegen/src/visit.rs create mode 100644 syn/codegen/src/visit_mut.rs create mode 100644 syn/dev/Cargo.toml create mode 100644 syn/dev/README.md create mode 100644 syn/dev/main.rs create mode 100644 syn/dev/parse.rs create mode 100644 syn/examples/README.md create mode 100644 syn/examples/dump-syntax/Cargo.toml create mode 100644 syn/examples/dump-syntax/README.md create mode 100644 syn/examples/dump-syntax/src/main.rs create mode 100644 syn/examples/heapsize/Cargo.toml create mode 100644 syn/examples/heapsize/README.md create mode 100644 syn/examples/heapsize/example/Cargo.toml create mode 100644 syn/examples/heapsize/example/src/main.rs create mode 100644 syn/examples/heapsize/heapsize/Cargo.toml create mode 100644 syn/examples/heapsize/heapsize/src/lib.rs create mode 100644 syn/examples/heapsize/heapsize_derive/Cargo.toml create mode 100644 syn/examples/heapsize/heapsize_derive/src/lib.rs create mode 100644 syn/examples/lazy-static/Cargo.toml create mode 100644 syn/examples/lazy-static/README.md create mode 100644 syn/examples/lazy-static/example/Cargo.toml create mode 100644 syn/examples/lazy-static/example/src/main.rs create mode 100644 syn/examples/lazy-static/lazy-static/Cargo.toml create mode 100644 syn/examples/lazy-static/lazy-static/src/lib.rs create mode 100644 syn/examples/trace-var/Cargo.toml create mode 100644 syn/examples/trace-var/README.md create mode 100644 syn/examples/trace-var/example/Cargo.toml create mode 100644 syn/examples/trace-var/example/src/main.rs create mode 100644 syn/examples/trace-var/trace-var/Cargo.toml create mode 100644 syn/examples/trace-var/trace-var/src/lib.rs create mode 100644 syn/json/Cargo.toml create mode 100644 syn/json/src/lib.rs create mode 100644 syn/src/attr.rs create mode 100644 syn/src/await.rs create mode 100644 syn/src/bigint.rs create mode 100644 syn/src/buffer.rs create mode 100644 syn/src/custom_keyword.rs create mode 100644 syn/src/custom_punctuation.rs create mode 100644 syn/src/data.rs create mode 100644 syn/src/derive.rs create mode 100644 syn/src/discouraged.rs create mode 100644 syn/src/error.rs create mode 100644 syn/src/export.rs create mode 100644 syn/src/expr.rs create mode 100644 syn/src/ext.rs create mode 100644 syn/src/file.rs create mode 100644 syn/src/gen/fold.rs create mode 100644 syn/src/gen/visit.rs create mode 100644 syn/src/gen/visit_mut.rs create mode 100644 syn/src/gen_helper.rs create mode 100644 syn/src/generics.rs create mode 100644 syn/src/group.rs create mode 100644 syn/src/ident.rs create mode 100644 syn/src/item.rs create mode 100644 syn/src/keyword.rs create mode 100644 syn/src/lib.rs create mode 100644 syn/src/lifetime.rs create mode 100644 syn/src/lit.rs create mode 100644 syn/src/lookahead.rs create mode 100644 syn/src/mac.rs create mode 100644 syn/src/macros.rs create mode 100644 syn/src/op.rs create mode 100644 syn/src/parse.rs create mode 100644 syn/src/parse_macro_input.rs create mode 100644 syn/src/parse_quote.rs create mode 100644 syn/src/pat.rs create mode 100644 syn/src/path.rs create mode 100644 syn/src/print.rs create mode 100644 syn/src/punctuated.rs create mode 100644 syn/src/sealed.rs create mode 100644 syn/src/span.rs create mode 100644 syn/src/spanned.rs create mode 100644 syn/src/stmt.rs create mode 100644 syn/src/thread.rs create mode 100644 syn/src/token.rs create mode 100644 syn/src/tt.rs create mode 100644 syn/src/ty.rs create mode 100644 syn/syn.json create mode 100644 syn/tests/common/eq.rs create mode 100644 syn/tests/common/mod.rs create mode 100644 syn/tests/common/parse.rs create mode 100644 syn/tests/debug/gen.rs create mode 100644 syn/tests/debug/mod.rs create mode 100644 syn/tests/features/error.rs create mode 100644 syn/tests/features/mod.rs create mode 100644 syn/tests/macros/mod.rs create mode 100644 syn/tests/repo/mod.rs create mode 100644 syn/tests/repo/progress.rs create mode 100644 syn/tests/test_asyncness.rs create mode 100644 syn/tests/test_attribute.rs create mode 100644 syn/tests/test_derive_input.rs create mode 100644 syn/tests/test_expr.rs create mode 100644 syn/tests/test_generics.rs create mode 100644 syn/tests/test_grouping.rs create mode 100644 syn/tests/test_ident.rs create mode 100644 syn/tests/test_iterators.rs create mode 100644 syn/tests/test_lit.rs create mode 100644 syn/tests/test_meta.rs create mode 100644 syn/tests/test_parse_buffer.rs create mode 100644 syn/tests/test_pat.rs create mode 100644 syn/tests/test_precedence.rs create mode 100644 syn/tests/test_receiver.rs create mode 100644 syn/tests/test_round_trip.rs create mode 100644 syn/tests/test_should_parse.rs create mode 100644 syn/tests/test_size.rs create mode 100644 syn/tests/test_token_trees.rs create mode 100644 syn/tests/test_visibility.rs create mode 100644 syn/tests/zzz_stable.rs (limited to 'syn') diff --git a/syn/.gitignore b/syn/.gitignore new file mode 100644 index 0000000..9fa6883 --- /dev/null +++ b/syn/.gitignore @@ -0,0 +1,3 @@ +target +Cargo.lock +tests/rust/* diff --git a/syn/.travis.yml b/syn/.travis.yml new file mode 100644 index 0000000..a8719cc --- /dev/null +++ b/syn/.travis.yml @@ -0,0 +1,76 @@ +language: rust + +before_script: + - set -o errexit + +script: + - shopt -s expand_aliases + - alias build="cargo build ${TARGET+--target=$TARGET}" + - build --no-default-features + - build + - build --features full + - build --features 'fold visit visit-mut' + - build --features 'full fold visit visit-mut' + - build --no-default-features --features derive + - build --no-default-features --features 'derive parsing' + - build --no-default-features --features 'derive printing' + - build --no-default-features --features 'proc-macro parsing printing' + - build --no-default-features --features full + - build --no-default-features --features 'full parsing' + - build --no-default-features --features 'full printing' + - build --no-default-features --features 'full parsing printing' + - build --no-default-features --features 'fold visit visit-mut parsing printing' + - build --no-default-features --features 'full fold visit visit-mut parsing printing' + +matrix: + include: + - rust: nightly + name: Tests + install: + - rustup component add rustc-dev + script: + - cargo test --all-features --release + - rust: nightly + - rust: stable + - rust: beta + - rust: 1.31.0 + - rust: nightly + name: Examples + script: + - cargo check --manifest-path examples/dump-syntax/Cargo.toml + - cargo check --manifest-path examples/heapsize/example/Cargo.toml + - cargo check --manifest-path examples/lazy-static/example/Cargo.toml + - cargo check --manifest-path examples/trace-var/example/Cargo.toml + - rust: nightly + name: Codegen + script: + - (cd codegen && cargo run) + - git diff --exit-code + - rust: nightly + name: Minimal versions + script: + - cargo update -Z minimal-versions + - cargo build --all-features + - rust: nightly + name: Clippy + script: + - rustup component add clippy || travis_terminate 0 + - cargo clippy --all-features + - rust: nightly + name: WebAssembly + env: TARGET=wasm32-unknown-unknown + install: + - rustup target add "${TARGET}" + - rust: nightly + name: WASI + env: TARGET=wasm32-wasi + install: + - rustup target add "${TARGET}" + allow_failures: + - rust: nightly + name: Clippy + fast_finish: true + +env: + global: + - RUST_MIN_STACK=20000000 diff --git a/syn/Cargo.toml b/syn/Cargo.toml new file mode 100644 index 0000000..11ad921 --- /dev/null +++ b/syn/Cargo.toml @@ -0,0 +1,72 @@ +[package] +name = "syn" +version = "1.0.12" # don't forget to update html_root_url and syn.json +authors = ["David Tolnay "] +license = "MIT OR Apache-2.0" +description = "Parser for Rust source code" +repository = "https://github.com/dtolnay/syn" +documentation = "https://docs.rs/syn" +categories = ["development-tools::procedural-macro-helpers"] +readme = "README.md" +include = [ + "/benches/**", + "/build.rs", + "/Cargo.toml", + "/LICENSE-APACHE", + "/LICENSE-MIT", + "/README.md", + "/src/**", + "/tests/**", +] +edition = "2018" + +[features] +default = ["derive", "parsing", "printing", "clone-impls", "proc-macro"] +derive = [] +full = [] +parsing = [] +printing = ["quote"] +visit = [] +visit-mut = [] +fold = [] +clone-impls = [] +extra-traits = [] +proc-macro = ["proc-macro2/proc-macro", "quote/proc-macro"] + +[dependencies] +proc-macro2 = { version = "1.0.7", default-features = false } +quote = { version = "1.0", optional = true, default-features = false } +unicode-xid = "0.2" + +[dev-dependencies] +anyhow = "1.0" +flate2 = "1.0" +insta = "0.12" +rayon = "1.0" +ref-cast = "1.0" +regex = "1.0" +reqwest = { version = "0.10", features = ["blocking"] } +tar = "0.4" +termcolor = "1.0" +walkdir = "2.1" + +[[bench]] +name = "rust" +harness = false +required-features = ["full", "parsing"] + +[[bench]] +name = "file" +required-features = ["full", "parsing"] + +[package.metadata.docs.rs] +all-features = true + +[package.metadata.playground] +all-features = true + +[badges] +travis-ci = { repository = "dtolnay/syn" } + +[workspace] +members = ["dev", "json"] diff --git a/syn/LICENSE-APACHE b/syn/LICENSE-APACHE new file mode 100644 index 0000000..16fe87b --- /dev/null +++ b/syn/LICENSE-APACHE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/syn/LICENSE-MIT b/syn/LICENSE-MIT new file mode 100644 index 0000000..31aa793 --- /dev/null +++ b/syn/LICENSE-MIT @@ -0,0 +1,23 @@ +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/syn/README.md b/syn/README.md new file mode 100644 index 0000000..29a7f32 --- /dev/null +++ b/syn/README.md @@ -0,0 +1,291 @@ +Parser for Rust source code +=========================== + +[![Build Status](https://api.travis-ci.org/dtolnay/syn.svg?branch=master)](https://travis-ci.org/dtolnay/syn) +[![Latest Version](https://img.shields.io/crates/v/syn.svg)](https://crates.io/crates/syn) +[![Rust Documentation](https://img.shields.io/badge/api-rustdoc-blue.svg)](https://docs.rs/syn/1.0/syn/) +[![Rustc Version 1.31+](https://img.shields.io/badge/rustc-1.31+-lightgray.svg)](https://blog.rust-lang.org/2018/12/06/Rust-1.31-and-rust-2018.html) + +Syn is a parsing library for parsing a stream of Rust tokens into a syntax tree +of Rust source code. + +Currently this library is geared toward use in Rust procedural macros, but +contains some APIs that may be useful more generally. + +- **Data structures** — Syn provides a complete syntax tree that can represent + any valid Rust source code. The syntax tree is rooted at [`syn::File`] which + represents a full source file, but there are other entry points that may be + useful to procedural macros including [`syn::Item`], [`syn::Expr`] and + [`syn::Type`]. + +- **Derives** — Of particular interest to derive macros is [`syn::DeriveInput`] + which is any of the three legal input items to a derive macro. An example + below shows using this type in a library that can derive implementations of a + user-defined trait. + +- **Parsing** — Parsing in Syn is built around [parser functions] with the + signature `fn(ParseStream) -> Result`. Every syntax tree node defined by + Syn is individually parsable and may be used as a building block for custom + syntaxes, or you may dream up your own brand new syntax without involving any + of our syntax tree types. + +- **Location information** — Every token parsed by Syn is associated with a + `Span` that tracks line and column information back to the source of that + token. These spans allow a procedural macro to display detailed error messages + pointing to all the right places in the user's code. There is an example of + this below. + +- **Feature flags** — Functionality is aggressively feature gated so your + procedural macros enable only what they need, and do not pay in compile time + for all the rest. + +[`syn::File`]: https://docs.rs/syn/1.0/syn/struct.File.html +[`syn::Item`]: https://docs.rs/syn/1.0/syn/enum.Item.html +[`syn::Expr`]: https://docs.rs/syn/1.0/syn/enum.Expr.html +[`syn::Type`]: https://docs.rs/syn/1.0/syn/enum.Type.html +[`syn::DeriveInput`]: https://docs.rs/syn/1.0/syn/struct.DeriveInput.html +[parser functions]: https://docs.rs/syn/1.0/syn/parse/index.html + +If you get stuck with anything involving procedural macros in Rust I am happy to +provide help even if the issue is not related to Syn. Please file a ticket in +this repo. + +*Version requirement: Syn supports rustc 1.31 and up.* + +[*Release notes*](https://github.com/dtolnay/syn/releases) + +
+ +## Resources + +The best way to learn about procedural macros is by writing some. Consider +working through [this procedural macro workshop][workshop] to get familiar with +the different types of procedural macros. The workshop contains relevant links +into the Syn documentation as you work through each project. + +[workshop]: https://github.com/dtolnay/proc-macro-workshop + +
+ +## Example of a derive macro + +The canonical derive macro using Syn looks like this. We write an ordinary Rust +function tagged with a `proc_macro_derive` attribute and the name of the trait +we are deriving. Any time that derive appears in the user's code, the Rust +compiler passes their data structure as tokens into our macro. We get to execute +arbitrary Rust code to figure out what to do with those tokens, then hand some +tokens back to the compiler to compile into the user's crate. + +[`TokenStream`]: https://doc.rust-lang.org/proc_macro/struct.TokenStream.html + +```toml +[dependencies] +syn = "1.0" +quote = "1.0" + +[lib] +proc-macro = true +``` + +```rust +extern crate proc_macro; + +use proc_macro::TokenStream; +use quote::quote; +use syn::{parse_macro_input, DeriveInput}; + +#[proc_macro_derive(MyMacro)] +pub fn my_macro(input: TokenStream) -> TokenStream { + // Parse the input tokens into a syntax tree + let input = parse_macro_input!(input as DeriveInput); + + // Build the output, possibly using quasi-quotation + let expanded = quote! { + // ... + }; + + // Hand the output tokens back to the compiler + TokenStream::from(expanded) +} +``` + +The [`heapsize`] example directory shows a complete working implementation of a +derive macro. It works on any Rust compiler 1.31+. The example derives a +`HeapSize` trait which computes an estimate of the amount of heap memory owned +by a value. + +[`heapsize`]: examples/heapsize + +```rust +pub trait HeapSize { + /// Total number of bytes of heap memory owned by `self`. + fn heap_size_of_children(&self) -> usize; +} +``` + +The derive macro allows users to write `#[derive(HeapSize)]` on data structures +in their program. + +```rust +#[derive(HeapSize)] +struct Demo<'a, T: ?Sized> { + a: Box, + b: u8, + c: &'a str, + d: String, +} +``` + +
+ +## Spans and error reporting + +The token-based procedural macro API provides great control over where the +compiler's error messages are displayed in user code. Consider the error the +user sees if one of their field types does not implement `HeapSize`. + +```rust +#[derive(HeapSize)] +struct Broken { + ok: String, + bad: std::thread::Thread, +} +``` + +By tracking span information all the way through the expansion of a procedural +macro as shown in the `heapsize` example, token-based macros in Syn are able to +trigger errors that directly pinpoint the source of the problem. + +``` +error[E0277]: the trait bound `std::thread::Thread: HeapSize` is not satisfied + --> src/main.rs:7:5 + | +7 | bad: std::thread::Thread, + | ^^^^^^^^^^^^^^^^^^^^^^^^ the trait `HeapSize` is not implemented for `std::thread::Thread` +``` + +
+ +## Parsing a custom syntax + +The [`lazy-static`] example directory shows the implementation of a +`functionlike!(...)` procedural macro in which the input tokens are parsed using +Syn's parsing API. + +[`lazy-static`]: examples/lazy-static + +The example reimplements the popular `lazy_static` crate from crates.io as a +procedural macro. + +``` +lazy_static! { + static ref USERNAME: Regex = Regex::new("^[a-z0-9_-]{3,16}$").unwrap(); +} +``` + +The implementation shows how to trigger custom warnings and error messages on +the macro input. + +``` +warning: come on, pick a more creative name + --> src/main.rs:10:16 + | +10 | static ref FOO: String = "lazy_static".to_owned(); + | ^^^ +``` + +
+ +## Testing + +When testing macros, we often care not just that the macro can be used +successfully but also that when the macro is provided with invalid input it +produces maximally helpful error messages. Consider using the [`trybuild`] crate +to write tests for errors that are emitted by your macro or errors detected by +the Rust compiler in the expanded code following misuse of the macro. Such tests +help avoid regressions from later refactors that mistakenly make an error no +longer trigger or be less helpful than it used to be. + +[`trybuild`]: https://github.com/dtolnay/trybuild + +
+ +## Debugging + +When developing a procedural macro it can be helpful to look at what the +generated code looks like. Use `cargo rustc -- -Zunstable-options +--pretty=expanded` or the [`cargo expand`] subcommand. + +[`cargo expand`]: https://github.com/dtolnay/cargo-expand + +To show the expanded code for some crate that uses your procedural macro, run +`cargo expand` from that crate. To show the expanded code for one of your own +test cases, run `cargo expand --test the_test_case` where the last argument is +the name of the test file without the `.rs` extension. + +This write-up by Brandon W Maister discusses debugging in more detail: +[Debugging Rust's new Custom Derive system][debugging]. + +[debugging]: https://quodlibetor.github.io/posts/debugging-rusts-new-custom-derive-system/ + +
+ +## Optional features + +Syn puts a lot of functionality behind optional features in order to optimize +compile time for the most common use cases. The following features are +available. + +- **`derive`** *(enabled by default)* — Data structures for representing the + possible input to a derive macro, including structs and enums and types. +- **`full`** — Data structures for representing the syntax tree of all valid + Rust source code, including items and expressions. +- **`parsing`** *(enabled by default)* — Ability to parse input tokens into a + syntax tree node of a chosen type. +- **`printing`** *(enabled by default)* — Ability to print a syntax tree node as + tokens of Rust source code. +- **`visit`** — Trait for traversing a syntax tree. +- **`visit-mut`** — Trait for traversing and mutating in place a syntax tree. +- **`fold`** — Trait for transforming an owned syntax tree. +- **`clone-impls`** *(enabled by default)* — Clone impls for all syntax tree + types. +- **`extra-traits`** — Debug, Eq, PartialEq, Hash impls for all syntax tree + types. +- **`proc-macro`** *(enabled by default)* — Runtime dependency on the dynamic + library libproc_macro from rustc toolchain. + +
+ +## Proc macro shim + +Syn operates on the token representation provided by the [proc-macro2] crate +from crates.io rather than using the compiler's built in proc-macro crate +directly. This enables code using Syn to execute outside of the context of a +procedural macro, such as in unit tests or build.rs, and we avoid needing +incompatible ecosystems for proc macros vs non-macro use cases. + +In general all of your code should be written against proc-macro2 rather than +proc-macro. The one exception is in the signatures of procedural macro entry +points, which are required by the language to use `proc_macro::TokenStream`. + +The proc-macro2 crate will automatically detect and use the compiler's data +structures when a procedural macro is active. + +[proc-macro2]: https://docs.rs/proc-macro2/1.0.0/proc_macro2/ + +
+ +#### License + + +Licensed under either of Apache License, Version +2.0 or MIT license at your option. + + +
+ + +Unless you explicitly state otherwise, any contribution intentionally submitted +for inclusion in this crate by you, as defined in the Apache-2.0 license, shall +be dual licensed as above, without any additional terms or conditions. + diff --git a/syn/appveyor.yml b/syn/appveyor.yml new file mode 100644 index 0000000..020c8ac --- /dev/null +++ b/syn/appveyor.yml @@ -0,0 +1,16 @@ +environment: + matrix: + - TARGET: x86_64-pc-windows-msvc + +install: + - ps: Start-FileDownload "https://static.rust-lang.org/dist/rust-nightly-${env:TARGET}.exe" + - rust-nightly-%TARGET%.exe /VERYSILENT /NORESTART /DIR="C:\Program Files (x86)\Rust" + - SET PATH=%PATH%;C:\Program Files (x86)\Rust\bin + - SET PATH=%PATH%;C:\MinGW\bin + - rustc -V + - cargo -V + +build: false + +test_script: + - cargo build --all-features diff --git a/syn/benches/file.rs b/syn/benches/file.rs new file mode 100644 index 0000000..58ab8df --- /dev/null +++ b/syn/benches/file.rs @@ -0,0 +1,30 @@ +// $ cargo bench --features full --bench file + +#![feature(rustc_private, test)] +#![recursion_limit = "1024"] + +extern crate test; + +#[macro_use] +#[path = "../tests/macros/mod.rs"] +mod macros; + +#[path = "../tests/common/mod.rs"] +mod common; +#[path = "../tests/repo/mod.rs"] +pub mod repo; + +use proc_macro2::TokenStream; +use std::fs; +use std::str::FromStr; +use test::Bencher; + +const FILE: &str = "tests/rust/src/libcore/str/mod.rs"; + +#[bench] +fn parse_file(b: &mut Bencher) { + repo::clone_rust(); + let content = fs::read_to_string(FILE).unwrap(); + let tokens = TokenStream::from_str(&content).unwrap(); + b.iter(|| syn::parse2::(tokens.clone())); +} diff --git a/syn/benches/rust.rs b/syn/benches/rust.rs new file mode 100644 index 0000000..941ecb9 --- /dev/null +++ b/syn/benches/rust.rs @@ -0,0 +1,158 @@ +// $ cargo bench --features full --bench rust +// +// Syn only, useful for profiling: +// $ RUSTFLAGS='--cfg syn_only' cargo build --release --features full --bench rust + +#![cfg_attr(not(syn_only), feature(rustc_private))] +#![recursion_limit = "1024"] + +#[macro_use] +#[path = "../tests/macros/mod.rs"] +mod macros; + +#[path = "../tests/common/mod.rs"] +mod common; +#[path = "../tests/repo/mod.rs"] +mod repo; + +use std::fs; +use std::time::{Duration, Instant}; + +#[cfg(not(syn_only))] +mod tokenstream_parse { + use proc_macro2::TokenStream; + use std::str::FromStr; + + pub fn bench(content: &str) -> Result<(), ()> { + TokenStream::from_str(content).map(drop).map_err(drop) + } +} + +mod syn_parse { + pub fn bench(content: &str) -> Result<(), ()> { + syn::parse_file(content).map(drop).map_err(drop) + } +} + +#[cfg(not(syn_only))] +mod libsyntax_parse { + extern crate rustc_data_structures; + extern crate rustc_parse; + extern crate rustc_span; + extern crate syntax; + + use rustc_data_structures::sync::Lrc; + use rustc_span::FileName; + use syntax::edition::Edition; + use syntax::errors::{emitter::Emitter, Diagnostic, Handler}; + use syntax::sess::ParseSess; + use syntax::source_map::{FilePathMapping, SourceMap}; + + pub fn bench(content: &str) -> Result<(), ()> { + struct SilentEmitter; + + impl Emitter for SilentEmitter { + fn emit_diagnostic(&mut self, _diag: &Diagnostic) {} + fn source_map(&self) -> Option<&Lrc> { + None + } + } + + syntax::with_globals(Edition::Edition2018, || { + let cm = Lrc::new(SourceMap::new(FilePathMapping::empty())); + let emitter = Box::new(SilentEmitter); + let handler = Handler::with_emitter(false, None, emitter); + let sess = ParseSess::with_span_handler(handler, cm); + if let Err(mut diagnostic) = rustc_parse::parse_crate_from_source_str( + FileName::Custom("bench".to_owned()), + content.to_owned(), + &sess, + ) { + diagnostic.cancel(); + return Err(()); + }; + Ok(()) + }) + } +} + +#[cfg(not(syn_only))] +mod read_from_disk { + pub fn bench(content: &str) -> Result<(), ()> { + let _ = content; + Ok(()) + } +} + +fn exec(mut codepath: impl FnMut(&str) -> Result<(), ()>) -> Duration { + let begin = Instant::now(); + let mut success = 0; + let mut total = 0; + + walkdir::WalkDir::new("tests/rust/src") + .into_iter() + .filter_entry(repo::base_dir_filter) + .for_each(|entry| { + let entry = entry.unwrap(); + let path = entry.path(); + if path.is_dir() { + return; + } + let content = fs::read_to_string(path).unwrap(); + let ok = codepath(&content).is_ok(); + success += ok as usize; + total += 1; + if !ok { + eprintln!("FAIL {}", path.display()); + } + }); + + assert_eq!(success, total); + begin.elapsed() +} + +fn main() { + repo::clone_rust(); + + macro_rules! testcases { + ($($(#[$cfg:meta])* $name:path,)*) => { + vec![ + $( + $(#[$cfg])* + (stringify!($name), $name as fn(&str) -> Result<(), ()>), + )* + ] + }; + } + + #[cfg(not(syn_only))] + { + let mut lines = 0; + let mut files = 0; + exec(|content| { + lines += content.lines().count(); + files += 1; + Ok(()) + }); + eprintln!("\n{} lines in {} files", lines, files); + } + + for (name, f) in testcases!( + #[cfg(not(syn_only))] + read_from_disk::bench, + #[cfg(not(syn_only))] + tokenstream_parse::bench, + syn_parse::bench, + #[cfg(not(syn_only))] + libsyntax_parse::bench, + ) { + eprint!("{:20}", format!("{}:", name)); + let elapsed = exec(f); + eprintln!( + "elapsed={}.{:03}s", + elapsed.as_secs(), + elapsed.subsec_millis(), + ); + } + eprintln!(); +} diff --git a/syn/build.rs b/syn/build.rs new file mode 100644 index 0000000..c6040c1 --- /dev/null +++ b/syn/build.rs @@ -0,0 +1,63 @@ +use std::env; +use std::process::Command; +use std::str::{self, FromStr}; + +// The rustc-cfg strings below are *not* public API. Please let us know by +// opening a GitHub issue if your build environment requires some way to enable +// these cfgs other than by executing our build script. +fn main() { + let compiler = match rustc_version() { + Some(compiler) => compiler, + None => return, + }; + + if compiler.minor < 36 { + println!("cargo:rustc-cfg=syn_omit_await_from_token_macro"); + } + + if !compiler.nightly { + println!("cargo:rustc-cfg=syn_disable_nightly_tests"); + } +} + +struct Compiler { + minor: u32, + nightly: bool, +} + +fn rustc_version() -> Option { + let rustc = match env::var_os("RUSTC") { + Some(rustc) => rustc, + None => return None, + }; + + let output = match Command::new(rustc).arg("--version").output() { + Ok(output) => output, + Err(_) => return None, + }; + + let version = match str::from_utf8(&output.stdout) { + Ok(version) => version, + Err(_) => return None, + }; + + let mut pieces = version.split('.'); + if pieces.next() != Some("rustc 1") { + return None; + } + + let next = match pieces.next() { + Some(next) => next, + None => return None, + }; + + let minor = match u32::from_str(next) { + Ok(minor) => minor, + Err(_) => return None, + }; + + Some(Compiler { + minor, + nightly: version.contains("nightly"), + }) +} diff --git a/syn/codegen/Cargo.toml b/syn/codegen/Cargo.toml new file mode 100644 index 0000000..44d890b --- /dev/null +++ b/syn/codegen/Cargo.toml @@ -0,0 +1,31 @@ +[package] +name = "syn-internal-codegen" +version = "0.0.0" +authors = ["David Tolnay ", "Nika Layzell "] +edition = "2018" + +publish = false # this is an internal crate which should never be published + +[dependencies] +anyhow = "1.0" +color-backtrace = "0.2" +indexmap = { version = "1.0", features = ["serde-1"] } +inflections = "1.1" +proc-macro2 = { version = "1.0", features = ["span-locations"] } +quote = "1.0" +rustfmt = { package = "rustfmt-nightly", git = "https://github.com/rust-lang-nursery/rustfmt" } +semver = { version = "0.9", features = ["serde"] } +serde = { version = "1.0.88", features = ["derive"] } +serde_json = "1.0.38" +syn-codegen = { path = "../json" } +syn = { path = "..", features = ["full", "extra-traits"] } +thiserror = "1.0" +toml = "0.4.10" + +# work around https://github.com/crossbeam-rs/crossbeam/issues/435 +# until https://github.com/BurntSushi/ripgrep/pull/1427 is released +crossbeam-utils = "=0.6.5" + +[workspace] +# Prefer that `cargo clean` in syn's directory does not require a rebuild of +# rustfmt in the codegen directory. diff --git a/syn/codegen/README.md b/syn/codegen/README.md new file mode 100644 index 0000000..df46bd2 --- /dev/null +++ b/syn/codegen/README.md @@ -0,0 +1,12 @@ +# syn_codegen + +This is an internal (not published on crates.io) crate which is used to generate +the files in the `gen/` directory of `syn`. It is used to ensure that the +implementations for `Fold`, `Visit`, and `VisitMut` remain in sync with the +actual AST. + +To run this program, run `cargo run` in this directory, and the `gen/` folder +will be re-generated. + +This program is slow, and is therefore not run when building `syn` as part of +the build script to save on compile time. diff --git a/syn/codegen/src/debug.rs b/syn/codegen/src/debug.rs new file mode 100644 index 0000000..9193881 --- /dev/null +++ b/syn/codegen/src/debug.rs @@ -0,0 +1,308 @@ +use crate::file; +use anyhow::Result; +use proc_macro2::{Ident, Span, TokenStream}; +use quote::quote; +use syn::Index; +use syn_codegen::{Data, Definitions, Node, Type}; + +const DEBUG_SRC: &str = "../tests/debug/gen.rs"; + +fn rust_type(ty: &Type) -> TokenStream { + match ty { + Type::Syn(ty) => { + let ident = Ident::new(ty, Span::call_site()); + quote!(syn::#ident) + } + Type::Std(ty) => { + let ident = Ident::new(ty, Span::call_site()); + quote!(#ident) + } + Type::Ext(ty) => { + let ident = Ident::new(ty, Span::call_site()); + quote!(proc_macro2::#ident) + } + Type::Token(ty) | Type::Group(ty) => { + let ident = Ident::new(ty, Span::call_site()); + quote!(syn::token::#ident) + } + Type::Punctuated(ty) => { + let element = rust_type(&ty.element); + let punct = Ident::new(&ty.punct, Span::call_site()); + quote!(syn::punctuated::Punctuated<#element, #punct>) + } + Type::Option(ty) => { + let inner = rust_type(ty); + quote!(Option<#inner>) + } + Type::Box(ty) => { + let inner = rust_type(ty); + quote!(Box<#inner>) + } + Type::Vec(ty) => { + let inner = rust_type(ty); + quote!(Vec<#inner>) + } + Type::Tuple(ty) => { + let inner = ty.iter().map(rust_type); + quote!((#(#inner,)*)) + } + } +} + +fn is_printable(ty: &Type) -> bool { + match ty { + Type::Ext(name) => name != "Span", + Type::Box(ty) => is_printable(ty), + Type::Tuple(ty) => ty.iter().any(is_printable), + Type::Token(_) | Type::Group(_) => false, + Type::Syn(name) => name != "Reserved", + Type::Std(_) | Type::Punctuated(_) | Type::Option(_) | Type::Vec(_) => true, + } +} + +fn format_field(val: &TokenStream, ty: &Type) -> Option { + if !is_printable(ty) { + return None; + } + let format = match ty { + Type::Option(ty) => { + let inner = quote!(_val); + let format = format_field(&inner, ty).map(|format| { + quote! { + formatter.write_str("(")?; + Debug::fmt(#format, formatter)?; + formatter.write_str(")")?; + } + }); + let ty = rust_type(ty); + quote!({ + #[derive(RefCast)] + #[repr(transparent)] + struct Print(Option<#ty>); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + match &self.0 { + Some(#inner) => { + formatter.write_str("Some")?; + #format + Ok(()) + } + None => formatter.write_str("None"), + } + } + } + Print::ref_cast(#val) + }) + } + Type::Tuple(ty) => { + let printable: Vec = ty + .iter() + .enumerate() + .filter_map(|(i, ty)| { + let index = Index::from(i); + let val = quote!(&#val.#index); + format_field(&val, ty) + }) + .collect(); + if printable.len() == 1 { + printable.into_iter().next().unwrap() + } else { + quote! { + &(#(#printable),*) + } + } + } + _ => quote! { Lite(#val) }, + }; + Some(format) +} + +fn syntax_tree_enum<'a>(outer: &str, inner: &str, fields: &'a [Type]) -> Option<&'a str> { + if fields.len() != 1 { + return None; + } + const WHITELIST: &[&str] = &["PathArguments", "Visibility"]; + match &fields[0] { + Type::Syn(ty) if WHITELIST.contains(&outer) || outer.to_owned() + inner == *ty => Some(ty), + _ => None, + } +} + +fn lookup<'a>(defs: &'a Definitions, name: &str) -> &'a Node { + for node in &defs.types { + if node.ident == name { + return node; + } + } + panic!("not found: {}", name) +} + +fn expand_impl_body(defs: &Definitions, node: &Node, name: &str) -> TokenStream { + let ident = Ident::new(&node.ident, Span::call_site()); + + match &node.data { + Data::Enum(variants) => { + let arms = variants.iter().map(|(v, fields)| { + let variant = Ident::new(v, Span::call_site()); + if fields.is_empty() { + quote! { + syn::#ident::#variant => formatter.write_str(#v), + } + } else if let Some(inner) = syntax_tree_enum(name, v, fields) { + let path = format!("{}::{}", name, v); + let format = expand_impl_body(defs, lookup(defs, inner), &path); + quote! { + syn::#ident::#variant(_val) => { + #format + } + } + } else if fields.len() == 1 { + let ty = &fields[0]; + let val = quote!(_val); + let format = format_field(&val, ty).map(|format| { + quote! { + formatter.write_str("(")?; + Debug::fmt(#format, formatter)?; + formatter.write_str(")")?; + } + }); + quote! { + syn::#ident::#variant(_val) => { + formatter.write_str(#v)?; + #format + Ok(()) + } + } + } else { + let pats = (0..fields.len()) + .map(|i| Ident::new(&format!("_v{}", i), Span::call_site())); + let fields = fields.iter().enumerate().filter_map(|(i, ty)| { + let index = Ident::new(&format!("_v{}", i), Span::call_site()); + let val = quote!(#index); + let format = format_field(&val, ty)?; + Some(quote! { + formatter.field(#format); + }) + }); + quote! { + syn::#ident::#variant(#(#pats),*) => { + let mut formatter = formatter.debug_tuple(#v); + #(#fields)* + formatter.finish() + } + } + } + }); + let nonexhaustive = if node.exhaustive { + None + } else { + Some(quote!(_ => unreachable!())) + }; + quote! { + match _val { + #(#arms)* + #nonexhaustive + } + } + } + Data::Struct(fields) => { + let fields = fields.iter().filter_map(|(f, ty)| { + let ident = Ident::new(f, Span::call_site()); + if let Type::Option(ty) = ty { + let inner = quote!(_val); + let format = format_field(&inner, ty).map(|format| { + quote! { + let #inner = &self.0; + formatter.write_str("(")?; + Debug::fmt(#format, formatter)?; + formatter.write_str(")")?; + } + }); + let ty = rust_type(ty); + Some(quote! { + if let Some(val) = &_val.#ident { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(#ty); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some")?; + #format + Ok(()) + } + } + formatter.field(#f, Print::ref_cast(val)); + } + }) + } else { + let val = quote!(&_val.#ident); + let format = format_field(&val, ty)?; + let mut call = quote! { + formatter.field(#f, #format); + }; + if let Type::Vec(_) | Type::Punctuated(_) = ty { + call = quote! { + if !_val.#ident.is_empty() { + #call + } + }; + } + Some(call) + } + }); + quote! { + let mut formatter = formatter.debug_struct(#name); + #(#fields)* + formatter.finish() + } + } + Data::Private => { + if node.ident == "LitInt" || node.ident == "LitFloat" { + quote! { + write!(formatter, "{}", _val) + } + } else { + quote! { + write!(formatter, "{:?}", _val.value()) + } + } + } + } +} + +fn expand_impl(defs: &Definitions, node: &Node) -> TokenStream { + if node.ident == "Reserved" { + return TokenStream::new(); + } + + let ident = Ident::new(&node.ident, Span::call_site()); + let body = expand_impl_body(defs, node, &node.ident); + + quote! { + impl Debug for Lite { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let _val = &self.value; + #body + } + } + } +} + +pub fn generate(defs: &Definitions) -> Result<()> { + let mut impls = TokenStream::new(); + for node in &defs.types { + impls.extend(expand_impl(&defs, node)); + } + + file::write( + DEBUG_SRC, + quote! { + use super::{Lite, RefCast}; + use std::fmt::{self, Debug}; + + #impls + }, + )?; + + Ok(()) +} diff --git a/syn/codegen/src/file.rs b/syn/codegen/src/file.rs new file mode 100644 index 0000000..5521d75 --- /dev/null +++ b/syn/codegen/src/file.rs @@ -0,0 +1,32 @@ +use anyhow::Result; +use proc_macro2::TokenStream; +use std::fs; +use std::io::Write; +use std::path::Path; + +pub fn write>(path: P, content: TokenStream) -> Result<()> { + let mut formatted = Vec::new(); + writeln!( + formatted, + "// This file is @generated by syn-internal-codegen." + )?; + writeln!(formatted, "// It is not intended for manual editing.")?; + writeln!(formatted)?; + + let mut config = rustfmt::Config::default(); + config.set().emit_mode(rustfmt::EmitMode::Stdout); + config.set().verbose(rustfmt::Verbosity::Quiet); + config.set().format_macro_matchers(true); + config.set().normalize_doc_attributes(true); + + let mut session = rustfmt::Session::new(config, Some(&mut formatted)); + session.format(rustfmt::Input::Text(content.to_string()))?; + drop(session); + + if path.as_ref().is_file() && fs::read(&path)? == formatted { + return Ok(()); + } + + fs::write(path, formatted)?; + Ok(()) +} diff --git a/syn/codegen/src/fold.rs b/syn/codegen/src/fold.rs new file mode 100644 index 0000000..6914d76 --- /dev/null +++ b/syn/codegen/src/fold.rs @@ -0,0 +1,284 @@ +use crate::{file, full, gen}; +use anyhow::Result; +use proc_macro2::{Ident, Span, TokenStream}; +use quote::quote; +use syn::Index; +use syn_codegen::{Data, Definitions, Features, Node, Type}; + +const FOLD_SRC: &str = "../src/gen/fold.rs"; + +fn simple_visit(item: &str, name: &TokenStream) -> TokenStream { + let ident = gen::under_name(item); + let method = Ident::new(&format!("fold_{}", ident), Span::call_site()); + quote! { + f.#method(#name) + } +} + +fn visit( + ty: &Type, + features: &Features, + defs: &Definitions, + name: &TokenStream, +) -> Option { + match ty { + Type::Box(t) => { + let res = visit(t, features, defs, "e!(*#name))?; + Some(quote! { + Box::new(#res) + }) + } + Type::Vec(t) => { + let operand = quote!(it); + let val = visit(t, features, defs, &operand)?; + Some(quote! { + FoldHelper::lift(#name, |it| { #val }) + }) + } + Type::Punctuated(p) => { + let operand = quote!(it); + let val = visit(&p.element, features, defs, &operand)?; + Some(quote! { + FoldHelper::lift(#name, |it| { #val }) + }) + } + Type::Option(t) => { + let it = quote!(it); + let val = visit(t, features, defs, &it)?; + Some(quote! { + (#name).map(|it| { #val }) + }) + } + Type::Tuple(t) => { + let mut code = TokenStream::new(); + for (i, elem) in t.iter().enumerate() { + let i = Index::from(i); + let it = quote!((#name).#i); + let val = visit(elem, features, defs, &it).unwrap_or(it); + code.extend(val); + code.extend(quote!(,)); + } + Some(quote! { + (#code) + }) + } + Type::Token(t) => { + let repr = &defs.tokens[t]; + let is_keyword = repr.chars().next().unwrap().is_alphabetic(); + let spans = if is_keyword { + quote!(span) + } else { + quote!(spans) + }; + let ty = if repr == "await" { + quote!(crate::token::Await) + } else { + syn::parse_str(&format!("Token![{}]", repr)).unwrap() + }; + Some(quote! { + #ty(tokens_helper(f, &#name.#spans)) + }) + } + Type::Group(t) => { + let ty = Ident::new(t, Span::call_site()); + Some(quote! { + #ty(tokens_helper(f, &#name.span)) + }) + } + Type::Syn(t) => { + fn requires_full(features: &Features) -> bool { + features.any.contains("full") && features.any.len() == 1 + } + let mut res = simple_visit(t, name); + let target = defs.types.iter().find(|ty| ty.ident == *t).unwrap(); + if requires_full(&target.features) && !requires_full(features) { + res = quote!(full!(#res)); + } + Some(res) + } + Type::Ext(t) if gen::TERMINAL_TYPES.contains(&&t[..]) => Some(simple_visit(t, name)), + Type::Ext(_) | Type::Std(_) => None, + } +} + +fn node(traits: &mut TokenStream, impls: &mut TokenStream, s: &Node, defs: &Definitions) { + let under_name = gen::under_name(&s.ident); + let ty = Ident::new(&s.ident, Span::call_site()); + let fold_fn = Ident::new(&format!("fold_{}", under_name), Span::call_site()); + + let mut fold_impl = TokenStream::new(); + + match &s.data { + Data::Enum(variants) => { + let mut fold_variants = TokenStream::new(); + + for (variant, fields) in variants { + let variant_ident = Ident::new(variant, Span::call_site()); + + if fields.is_empty() { + fold_variants.extend(quote! { + #ty::#variant_ident => { + #ty::#variant_ident + } + }); + } else { + let mut bind_fold_fields = TokenStream::new(); + let mut fold_fields = TokenStream::new(); + + for (idx, ty) in fields.iter().enumerate() { + let name = format!("_binding_{}", idx); + let binding = Ident::new(&name, Span::call_site()); + + bind_fold_fields.extend(quote! { + #binding, + }); + + let owned_binding = quote!(#binding); + + fold_fields.extend( + visit(ty, &s.features, defs, &owned_binding).unwrap_or(owned_binding), + ); + + fold_fields.extend(quote!(,)); + } + + fold_variants.extend(quote! { + #ty::#variant_ident(#bind_fold_fields) => { + #ty::#variant_ident( + #fold_fields + ) + } + }); + } + } + + let nonexhaustive = if s.exhaustive { + None + } else { + Some(quote!(_ => unreachable!())) + }; + + fold_impl.extend(quote! { + match node { + #fold_variants + #nonexhaustive + } + }); + } + Data::Struct(fields) => { + let mut fold_fields = TokenStream::new(); + + for (field, ty) in fields { + let id = Ident::new(&field, Span::call_site()); + let ref_toks = quote!(node.#id); + + if let Type::Syn(ty) = ty { + if ty == "Reserved" { + fold_fields.extend(quote! { + #id: #ref_toks, + }); + continue; + } + } + + let fold = visit(&ty, &s.features, defs, &ref_toks).unwrap_or(ref_toks); + + fold_fields.extend(quote! { + #id: #fold, + }); + } + + if !fields.is_empty() { + fold_impl.extend(quote! { + #ty { + #fold_fields + } + }) + } else { + if ty == "Ident" { + fold_impl.extend(quote! { + let mut node = node; + let span = f.fold_span(node.span()); + node.set_span(span); + }); + } + fold_impl.extend(quote! { + node + }); + } + } + Data::Private => { + if ty == "Ident" { + fold_impl.extend(quote! { + let mut node = node; + let span = f.fold_span(node.span()); + node.set_span(span); + }); + } + fold_impl.extend(quote! { + node + }); + } + } + + let fold_span_only = + s.data == Data::Private && !gen::TERMINAL_TYPES.contains(&s.ident.as_str()); + if fold_span_only { + fold_impl = quote! { + let span = f.fold_span(node.span()); + let mut node = node; + node.set_span(span); + node + }; + } + + traits.extend(quote! { + fn #fold_fn(&mut self, i: #ty) -> #ty { + #fold_fn(self, i) + } + }); + + impls.extend(quote! { + pub fn #fold_fn(f: &mut F, node: #ty) -> #ty + where + F: Fold + ?Sized, + { + #fold_impl + } + }); +} + +pub fn generate(defs: &Definitions) -> Result<()> { + let (traits, impls) = gen::traverse(defs, node); + let full_macro = full::get_macro(); + file::write( + FOLD_SRC, + quote! { + // Unreachable code is generated sometimes without the full feature. + #![allow(unreachable_code, unused_variables)] + + use crate::*; + #[cfg(any(feature = "full", feature = "derive"))] + use crate::token::{Brace, Bracket, Paren, Group}; + use proc_macro2::Span; + #[cfg(any(feature = "full", feature = "derive"))] + use crate::gen::helper::fold::*; + + #full_macro + + /// Syntax tree traversal to transform the nodes of an owned syntax tree. + /// + /// See the [module documentation] for details. + /// + /// [module documentation]: self + /// + /// *This trait is available if Syn is built with the `"fold"` feature.* + pub trait Fold { + #traits + } + + #impls + }, + )?; + Ok(()) +} diff --git a/syn/codegen/src/full.rs b/syn/codegen/src/full.rs new file mode 100644 index 0000000..a410031 --- /dev/null +++ b/syn/codegen/src/full.rs @@ -0,0 +1,20 @@ +use proc_macro2::TokenStream; +use quote::quote; + +pub fn get_macro() -> TokenStream { + quote! { + #[cfg(feature = "full")] + macro_rules! full { + ($e:expr) => { + $e + }; + } + + #[cfg(all(feature = "derive", not(feature = "full")))] + macro_rules! full { + ($e:expr) => { + unreachable!() + }; + } + } +} diff --git a/syn/codegen/src/gen.rs b/syn/codegen/src/gen.rs new file mode 100644 index 0000000..ef43182 --- /dev/null +++ b/syn/codegen/src/gen.rs @@ -0,0 +1,45 @@ +use inflections::Inflect; +use proc_macro2::{Ident, Span, TokenStream}; +use quote::quote; +use syn_codegen::{Data, Definitions, Features, Node}; + +pub const TERMINAL_TYPES: &[&str] = &["Span", "Ident"]; + +pub fn under_name(name: &str) -> Ident { + Ident::new(&name.to_snake_case(), Span::call_site()) +} + +pub fn traverse( + defs: &Definitions, + node: fn(&mut TokenStream, &mut TokenStream, &Node, &Definitions), +) -> (TokenStream, TokenStream) { + let mut types = defs.types.clone(); + for terminal in TERMINAL_TYPES { + types.push(Node { + ident: terminal.to_string(), + features: Features::default(), + data: Data::Private, + exhaustive: true, + }); + } + types.sort_by(|a, b| a.ident.cmp(&b.ident)); + + let mut traits = TokenStream::new(); + let mut impls = TokenStream::new(); + for s in types { + if s.ident == "Reserved" { + continue; + } + let features = &s.features.any; + let features = match features.len() { + 0 => quote!(), + 1 => quote!(#[cfg(feature = #(#features)*)]), + _ => quote!(#[cfg(any(#(feature = #features),*))]), + }; + traits.extend(features.clone()); + impls.extend(features); + node(&mut traits, &mut impls, &s, defs); + } + + (traits, impls) +} diff --git a/syn/codegen/src/json.rs b/syn/codegen/src/json.rs new file mode 100644 index 0000000..53904bf --- /dev/null +++ b/syn/codegen/src/json.rs @@ -0,0 +1,18 @@ +use anyhow::Result; +use std::fs; +use std::path::Path; +use syn_codegen::Definitions; + +pub fn generate(defs: &Definitions) -> Result<()> { + let mut j = serde_json::to_string_pretty(&defs)?; + j.push('\n'); + + let check: Definitions = serde_json::from_str(&j)?; + assert_eq!(*defs, check); + + let codegen_root = Path::new(env!("CARGO_MANIFEST_DIR")); + let json_path = codegen_root.join("../syn.json"); + fs::write(json_path, j)?; + + Ok(()) +} diff --git a/syn/codegen/src/main.rs b/syn/codegen/src/main.rs new file mode 100644 index 0000000..9b1b5dd --- /dev/null +++ b/syn/codegen/src/main.rs @@ -0,0 +1,36 @@ +// This crate crawls the Syn source directory to find all structs and enums that +// form the Syn syntax tree. +// +// A machine-readable representation of the syntax tree is saved to syn.json in +// the repo root for other code generation tools to consume. The syn-codegen +// crate (https://docs.rs/syn-codegen/) provides the data structures for parsing +// and making use of syn.json from Rust code. +// +// Finally this crate generates the Visit, VisitMut, and Fold traits in Syn +// programmatically from the syntax tree description. + +#![recursion_limit = "128"] +#![allow(clippy::needless_pass_by_value)] + +mod debug; +mod file; +mod fold; +mod full; +mod gen; +mod json; +mod operand; +mod parse; +mod version; +mod visit; +mod visit_mut; + +fn main() -> anyhow::Result<()> { + color_backtrace::install(); + let defs = parse::parse()?; + json::generate(&defs)?; + fold::generate(&defs)?; + visit::generate(&defs)?; + visit_mut::generate(&defs)?; + debug::generate(&defs)?; + Ok(()) +} diff --git a/syn/codegen/src/operand.rs b/syn/codegen/src/operand.rs new file mode 100644 index 0000000..db3bd18 --- /dev/null +++ b/syn/codegen/src/operand.rs @@ -0,0 +1,38 @@ +use proc_macro2::TokenStream; +use quote::quote; + +pub enum Operand { + Borrowed(TokenStream), + Owned(TokenStream), +} + +pub use self::Operand::*; + +impl Operand { + pub fn tokens(&self) -> &TokenStream { + match self { + Borrowed(n) | Owned(n) => n, + } + } + + pub fn ref_tokens(&self) -> TokenStream { + match self { + Borrowed(n) => n.clone(), + Owned(n) => quote!(&#n), + } + } + + pub fn ref_mut_tokens(&self) -> TokenStream { + match self { + Borrowed(n) => n.clone(), + Owned(n) => quote!(&mut #n), + } + } + + pub fn owned_tokens(&self) -> TokenStream { + match self { + Borrowed(n) => quote!(*#n), + Owned(n) => n.clone(), + } + } +} diff --git a/syn/codegen/src/parse.rs b/syn/codegen/src/parse.rs new file mode 100644 index 0000000..cdd6085 --- /dev/null +++ b/syn/codegen/src/parse.rs @@ -0,0 +1,657 @@ +use crate::version; +use anyhow::{bail, Result}; +use indexmap::IndexMap; +use quote::quote; +use syn::parse::Parser; +use syn::{parse_quote, Data, DataStruct, DeriveInput, Ident, Item}; +use syn_codegen as types; +use thiserror::Error; + +use std::collections::BTreeMap; +use std::fs; +use std::path::{Path, PathBuf}; + +const SYN_CRATE_ROOT: &str = "../src/lib.rs"; +const TOKEN_SRC: &str = "../src/token.rs"; +const IGNORED_MODS: &[&str] = &["fold", "visit", "visit_mut"]; +const EXTRA_TYPES: &[&str] = &["Lifetime"]; +const NONEXHAUSTIVE: &str = "__Nonexhaustive"; + +// NOTE: BTreeMap is used here instead of HashMap to have deterministic output. +type ItemLookup = BTreeMap; +type TokenLookup = BTreeMap; + +/// Parse the contents of `src` and return a list of AST types. +pub fn parse() -> Result { + let mut item_lookup = BTreeMap::new(); + load_file(SYN_CRATE_ROOT, &[], &mut item_lookup)?; + + let token_lookup = load_token_file(TOKEN_SRC)?; + + let version = version::get()?; + + let types = item_lookup + .values() + .map(|item| introspect_item(item, &item_lookup, &token_lookup)) + .collect(); + + let tokens = token_lookup + .into_iter() + .map(|(name, ty)| (ty, name)) + .collect(); + + Ok(types::Definitions { + version, + types, + tokens, + }) +} + +/// Data extracted from syn source +#[derive(Clone)] +pub struct AstItem { + ast: DeriveInput, + features: Vec, +} + +fn introspect_item(item: &AstItem, items: &ItemLookup, tokens: &TokenLookup) -> types::Node { + let features = introspect_features(&item.features); + + match &item.ast.data { + Data::Enum(ref data) => types::Node { + ident: item.ast.ident.to_string(), + features, + data: types::Data::Enum(introspect_enum(data, items, tokens)), + exhaustive: data.variants.iter().all(|v| v.ident != NONEXHAUSTIVE), + }, + Data::Struct(ref data) => types::Node { + ident: item.ast.ident.to_string(), + features, + data: { + if data.fields.iter().all(|f| is_pub(&f.vis)) { + types::Data::Struct(introspect_struct(data, items, tokens)) + } else { + types::Data::Private + } + }, + exhaustive: true, + }, + Data::Union(..) => panic!("Union not supported"), + } +} + +fn introspect_enum( + item: &syn::DataEnum, + items: &ItemLookup, + tokens: &TokenLookup, +) -> types::Variants { + item.variants + .iter() + .filter_map(|variant| { + if variant.ident == NONEXHAUSTIVE { + return None; + } + let fields = match &variant.fields { + syn::Fields::Unnamed(fields) => fields + .unnamed + .iter() + .map(|field| introspect_type(&field.ty, items, tokens)) + .collect(), + syn::Fields::Unit => vec![], + _ => panic!("Enum representation not supported"), + }; + Some((variant.ident.to_string(), fields)) + }) + .collect() +} + +fn introspect_struct( + item: &syn::DataStruct, + items: &ItemLookup, + tokens: &TokenLookup, +) -> types::Fields { + match &item.fields { + syn::Fields::Named(fields) => fields + .named + .iter() + .map(|field| { + ( + field.ident.as_ref().unwrap().to_string(), + introspect_type(&field.ty, items, tokens), + ) + }) + .collect(), + syn::Fields::Unit => IndexMap::new(), + _ => panic!("Struct representation not supported"), + } +} + +fn introspect_type(item: &syn::Type, items: &ItemLookup, tokens: &TokenLookup) -> types::Type { + match item { + syn::Type::Path(syn::TypePath { + qself: None, + ref path, + }) => { + let last = path.segments.last().unwrap(); + let string = last.ident.to_string(); + + match string.as_str() { + "Option" => { + let nested = introspect_type(first_arg(&last.arguments), items, tokens); + types::Type::Option(Box::new(nested)) + } + "Punctuated" => { + let nested = introspect_type(first_arg(&last.arguments), items, tokens); + let punct = match introspect_type(last_arg(&last.arguments), items, tokens) { + types::Type::Token(s) => s, + _ => panic!(), + }; + + types::Type::Punctuated(types::Punctuated { + element: Box::new(nested), + punct, + }) + } + "Vec" => { + let nested = introspect_type(first_arg(&last.arguments), items, tokens); + types::Type::Vec(Box::new(nested)) + } + "Box" => { + let nested = introspect_type(first_arg(&last.arguments), items, tokens); + types::Type::Box(Box::new(nested)) + } + "Brace" | "Bracket" | "Paren" | "Group" => types::Type::Group(string), + "TokenStream" | "Literal" | "Ident" | "Span" => types::Type::Ext(string), + "String" | "u32" | "usize" | "bool" => types::Type::Std(string), + "Await" => types::Type::Token("Await".to_string()), + _ => { + if items.get(&last.ident).is_some() || last.ident == "Reserved" { + types::Type::Syn(string) + } else { + unimplemented!("{}", string); + } + } + } + } + syn::Type::Tuple(syn::TypeTuple { ref elems, .. }) => { + let tys = elems + .iter() + .map(|ty| introspect_type(&ty, items, tokens)) + .collect(); + types::Type::Tuple(tys) + } + syn::Type::Macro(syn::TypeMacro { ref mac }) + if mac.path.segments.last().unwrap().ident == "Token" => + { + let content = mac.tokens.to_string(); + let ty = tokens.get(&content).unwrap().to_string(); + + types::Type::Token(ty) + } + _ => panic!("{}", quote!(#item).to_string()), + } +} + +fn introspect_features(attrs: &[syn::Attribute]) -> types::Features { + let mut ret = types::Features::default(); + + for attr in attrs { + if !attr.path.is_ident("cfg") { + continue; + } + + let features = parsing::parse_features.parse2(attr.tokens.clone()).unwrap(); + + if ret.any.is_empty() { + ret = features; + } else if ret.any.len() < features.any.len() { + assert!(ret.any.iter().all(|f| features.any.contains(f))); + } else { + assert!(features.any.iter().all(|f| ret.any.contains(f))); + ret = features; + } + } + + ret +} + +fn is_pub(vis: &syn::Visibility) -> bool { + match vis { + syn::Visibility::Public(_) => true, + _ => false, + } +} + +fn first_arg(params: &syn::PathArguments) -> &syn::Type { + let data = match *params { + syn::PathArguments::AngleBracketed(ref data) => data, + _ => panic!("Expected at least 1 type argument here"), + }; + + match *data + .args + .first() + .expect("Expected at least 1 type argument here") + { + syn::GenericArgument::Type(ref ty) => ty, + _ => panic!("Expected at least 1 type argument here"), + } +} + +fn last_arg(params: &syn::PathArguments) -> &syn::Type { + let data = match *params { + syn::PathArguments::AngleBracketed(ref data) => data, + _ => panic!("Expected at least 1 type argument here"), + }; + + match *data + .args + .last() + .expect("Expected at least 1 type argument here") + { + syn::GenericArgument::Type(ref ty) => ty, + _ => panic!("Expected at least 1 type argument here"), + } +} + +mod parsing { + use super::{AstItem, TokenLookup}; + + use proc_macro2::{TokenStream, TokenTree}; + use quote::quote; + use syn; + use syn::parse::{ParseStream, Result}; + use syn::*; + use syn_codegen as types; + + use std::collections::{BTreeMap, BTreeSet}; + + fn peek_tag(input: ParseStream, tag: &str) -> bool { + let ahead = input.fork(); + ahead.parse::().is_ok() + && ahead + .parse::() + .map(|ident| ident == tag) + .unwrap_or(false) + } + + // Parses #full - returns #[cfg(feature = "full")] if it is present, and + // nothing otherwise. + fn full(input: ParseStream) -> Vec { + if peek_tag(input, "full") { + input.parse::().unwrap(); + input.parse::().unwrap(); + vec![parse_quote!(#[cfg(feature = "full")])] + } else { + vec![] + } + } + + fn skip_manual_extra_traits(input: ParseStream) { + if peek_tag(input, "manual_extra_traits") || peek_tag(input, "manual_extra_traits_debug") { + input.parse::().unwrap(); + input.parse::().unwrap(); + } + } + + // Parses a simple AstStruct without the `pub struct` prefix. + fn ast_struct_inner(input: ParseStream) -> Result { + let ident: Ident = input.parse()?; + let features = full(input); + skip_manual_extra_traits(input); + let rest: TokenStream = input.parse()?; + Ok(AstItem { + ast: syn::parse2(quote! { + pub struct #ident #rest + })?, + features, + }) + } + + pub fn ast_struct(input: ParseStream) -> Result { + input.call(Attribute::parse_outer)?; + input.parse::()?; + input.parse::()?; + let res = input.call(ast_struct_inner)?; + Ok(res) + } + + fn no_visit(input: ParseStream) -> bool { + if peek_tag(input, "no_visit") { + input.parse::().unwrap(); + input.parse::().unwrap(); + true + } else { + false + } + } + + pub fn ast_enum(input: ParseStream) -> Result> { + input.call(Attribute::parse_outer)?; + input.parse::()?; + input.parse::()?; + let ident: Ident = input.parse()?; + skip_manual_extra_traits(input); + let no_visit = no_visit(input); + let rest: TokenStream = input.parse()?; + Ok(if no_visit { + None + } else { + Some(AstItem { + ast: syn::parse2(quote! { + pub enum #ident #rest + })?, + features: vec![], + }) + }) + } + + // A single variant of an ast_enum_of_structs! + struct EosVariant { + name: Ident, + member: Option, + } + fn eos_variant(input: ParseStream) -> Result { + input.call(Attribute::parse_outer)?; + let variant: Ident = input.parse()?; + let member = if input.peek(token::Paren) { + let content; + parenthesized!(content in input); + let path: Path = content.parse()?; + Some(path) + } else { + None + }; + input.parse::()?; + Ok(EosVariant { + name: variant, + member, + }) + } + + pub fn ast_enum_of_structs(input: ParseStream) -> Result { + input.call(Attribute::parse_outer)?; + input.parse::()?; + input.parse::()?; + let ident: Ident = input.parse()?; + skip_manual_extra_traits(input); + + let content; + braced!(content in input); + let mut variants = Vec::new(); + while !content.is_empty() { + variants.push(content.call(eos_variant)?); + } + + if let Some(ident) = input.parse::>()? { + assert_eq!(ident, "do_not_generate_to_tokens"); + } + + let enum_item = { + let variants = variants.iter().map(|v| { + let name = v.name.clone(); + match v.member { + Some(ref member) => quote!(#name(#member)), + None => quote!(#name), + } + }); + parse_quote! { + pub enum #ident { + #(#variants),* + } + } + }; + Ok(AstItem { + ast: enum_item, + features: vec![], + }) + } + + mod kw { + syn::custom_keyword!(macro_rules); + syn::custom_keyword!(Token); + } + + pub fn parse_token_macro(input: ParseStream) -> Result { + input.parse::()?; + input.parse::]>()?; + + let definition; + braced!(definition in input); + definition.call(Attribute::parse_outer)?; + definition.parse::()?; + definition.parse::()?; + definition.parse::()?; + + let rules; + braced!(rules in definition); + input.parse::()?; + + let mut tokens = BTreeMap::new(); + while !rules.is_empty() { + if rules.peek(Token![$]) { + rules.parse::()?; + rules.parse::()?; + rules.parse::()?; + tokens.insert("await".to_owned(), "Await".to_owned()); + } else { + let pattern; + parenthesized!(pattern in rules); + let token = pattern.parse::()?.to_string(); + rules.parse::]>()?; + let expansion; + braced!(expansion in rules); + rules.parse::()?; + expansion.parse::()?; + let path: Path = expansion.parse()?; + let ty = path.segments.last().unwrap().ident.to_string(); + tokens.insert(token, ty.to_string()); + } + } + Ok(tokens) + } + + fn parse_feature(input: ParseStream) -> Result { + let i: syn::Ident = input.parse()?; + assert_eq!(i, "feature"); + + input.parse::()?; + let s = input.parse::()?; + + Ok(s.value()) + } + + pub fn parse_features(input: ParseStream) -> Result { + let mut features = BTreeSet::new(); + + let level_1; + parenthesized!(level_1 in input); + + let i: syn::Ident = level_1.fork().parse()?; + + if i == "any" { + level_1.parse::()?; + + let level_2; + parenthesized!(level_2 in level_1); + + while !level_2.is_empty() { + features.insert(parse_feature(&level_2)?); + + if !level_2.is_empty() { + level_2.parse::()?; + } + } + } else if i == "feature" { + features.insert(parse_feature(&level_1)?); + assert!(level_1.is_empty()); + } else { + panic!("{:?}", i); + } + + assert!(input.is_empty()); + + Ok(types::Features { any: features }) + } +} + +fn get_features(attrs: &[syn::Attribute], base: &[syn::Attribute]) -> Vec { + let mut ret = base.to_owned(); + + for attr in attrs { + if attr.path.is_ident("cfg") { + ret.push(attr.clone()); + } + } + + ret +} + +#[derive(Error, Debug)] +#[error("{path}:{line}:{column}: {error}")] +struct LoadFileError { + path: PathBuf, + line: usize, + column: usize, + error: syn::Error, +} + +fn load_file>( + name: P, + features: &[syn::Attribute], + lookup: &mut ItemLookup, +) -> Result<()> { + let error = match do_load_file(&name, features, lookup).err() { + None => return Ok(()), + Some(error) => error, + }; + + let error = error.downcast::()?; + let span = error.span().start(); + + bail!(LoadFileError { + path: name.as_ref().to_owned(), + line: span.line, + column: span.column + 1, + error, + }) +} + +fn do_load_file>( + name: P, + features: &[syn::Attribute], + lookup: &mut ItemLookup, +) -> Result<()> { + let name = name.as_ref(); + let parent = name.parent().expect("no parent path"); + + // Parse the file + let src = fs::read_to_string(name)?; + let file = syn::parse_file(&src)?; + + // Collect all of the interesting AstItems declared in this file or submodules. + 'items: for item in file.items { + match item { + Item::Mod(item) => { + // Don't inspect inline modules. + if item.content.is_some() { + continue; + } + + // We don't want to try to load the generated rust files and + // parse them, so we ignore them here. + for name in IGNORED_MODS { + if item.ident == name { + continue 'items; + } + } + + // Lookup any #[cfg()] attributes on the module and add them to + // the feature set. + // + // The derive module is weird because it is built with either + // `full` or `derive` but exported only under `derive`. + let features = if item.ident == "derive" { + vec![parse_quote!(#[cfg(feature = "derive")])] + } else { + get_features(&item.attrs, features) + }; + + // Look up the submodule file, and recursively parse it. + // Only handles same-directory .rs file submodules for now. + let path = parent.join(&format!("{}.rs", item.ident)); + load_file(path, &features, lookup)?; + } + Item::Macro(item) => { + // Lookip any #[cfg()] attributes directly on the macro + // invocation, and add them to the feature set. + let features = get_features(&item.attrs, features); + + // Try to parse the AstItem declaration out of the item. + let tts = item.mac.tokens.clone(); + let found = if item.mac.path.is_ident("ast_struct") { + Some(parsing::ast_struct.parse2(tts)?) + } else if item.mac.path.is_ident("ast_enum") { + parsing::ast_enum.parse2(tts)? + } else if item.mac.path.is_ident("ast_enum_of_structs") { + Some(parsing::ast_enum_of_structs.parse2(tts)?) + } else { + continue; + }; + + // Record our features on the parsed AstItems. + for mut item in found { + if item.ast.ident != "Reserved" { + item.features.extend(features.clone()); + lookup.insert(item.ast.ident.clone(), item); + } + } + } + Item::Struct(item) => { + let ident = item.ident; + if EXTRA_TYPES.contains(&&ident.to_string()[..]) { + lookup.insert( + ident.clone(), + AstItem { + ast: DeriveInput { + ident, + vis: item.vis, + attrs: item.attrs, + generics: item.generics, + data: Data::Struct(DataStruct { + fields: item.fields, + struct_token: item.struct_token, + semi_token: item.semi_token, + }), + }, + features: features.to_owned(), + }, + ); + } + } + _ => {} + } + } + Ok(()) +} + +fn load_token_file>(name: P) -> Result { + let name = name.as_ref(); + let src = fs::read_to_string(name)?; + let file = syn::parse_file(&src)?; + for item in file.items { + match item { + Item::Macro(item) => { + match item.ident { + Some(ref i) if i == "export_token_macro" => {} + _ => continue, + } + let tokens = item.mac.parse_body_with(parsing::parse_token_macro)?; + return Ok(tokens); + } + _ => {} + } + } + + panic!("failed to parse Token macro") +} diff --git a/syn/codegen/src/version.rs b/syn/codegen/src/version.rs new file mode 100644 index 0000000..9374624 --- /dev/null +++ b/syn/codegen/src/version.rs @@ -0,0 +1,24 @@ +use anyhow::Result; +use semver::Version; +use serde::Deserialize; + +use std::fs; +use std::path::Path; + +pub fn get() -> Result { + let codegen_root = Path::new(env!("CARGO_MANIFEST_DIR")); + let syn_cargo_toml = codegen_root.join("../Cargo.toml"); + let manifest = fs::read_to_string(syn_cargo_toml)?; + let parsed: Manifest = toml::from_str(&manifest)?; + Ok(parsed.package.version) +} + +#[derive(Debug, Deserialize)] +struct Manifest { + package: Package, +} + +#[derive(Debug, Deserialize)] +struct Package { + version: Version, +} diff --git a/syn/codegen/src/visit.rs b/syn/codegen/src/visit.rs new file mode 100644 index 0000000..41bc9e9 --- /dev/null +++ b/syn/codegen/src/visit.rs @@ -0,0 +1,265 @@ +use crate::operand::{Borrowed, Operand, Owned}; +use crate::{file, full, gen}; +use anyhow::Result; +use proc_macro2::{Ident, Span, TokenStream}; +use quote::quote; +use syn::Index; +use syn_codegen::{Data, Definitions, Features, Node, Type}; + +const VISIT_SRC: &str = "../src/gen/visit.rs"; + +fn simple_visit(item: &str, name: &Operand) -> TokenStream { + let ident = gen::under_name(item); + let method = Ident::new(&format!("visit_{}", ident), Span::call_site()); + let name = name.ref_tokens(); + quote! { + v.#method(#name) + } +} + +fn noop_visit(name: &Operand) -> TokenStream { + let name = name.tokens(); + quote! { + skip!(#name) + } +} + +fn visit( + ty: &Type, + features: &Features, + defs: &Definitions, + name: &Operand, +) -> Option { + match ty { + Type::Box(t) => { + let name = name.owned_tokens(); + visit(t, features, defs, &Owned(quote!(*#name))) + } + Type::Vec(t) => { + let operand = Borrowed(quote!(it)); + let val = visit(t, features, defs, &operand)?; + let name = name.ref_tokens(); + Some(quote! { + for it in #name { + #val + } + }) + } + Type::Punctuated(p) => { + let operand = Borrowed(quote!(it)); + let val = visit(&p.element, features, defs, &operand)?; + let name = name.ref_tokens(); + Some(quote! { + for el in Punctuated::pairs(#name) { + let (it, p) = el.into_tuple(); + #val; + if let Some(p) = p { + tokens_helper(v, &p.spans); + } + } + }) + } + Type::Option(t) => { + let it = Borrowed(quote!(it)); + let val = visit(t, features, defs, &it)?; + let name = name.owned_tokens(); + Some(quote! { + if let Some(it) = &#name { + #val + } + }) + } + Type::Tuple(t) => { + let mut code = TokenStream::new(); + for (i, elem) in t.iter().enumerate() { + let name = name.tokens(); + let i = Index::from(i); + let it = Owned(quote!((#name).#i)); + let val = visit(elem, features, defs, &it).unwrap_or_else(|| noop_visit(&it)); + code.extend(val); + code.extend(quote!(;)); + } + Some(code) + } + Type::Token(t) => { + let name = name.tokens(); + let repr = &defs.tokens[t]; + let is_keyword = repr.chars().next().unwrap().is_alphabetic(); + let spans = if is_keyword { + quote!(span) + } else { + quote!(spans) + }; + Some(quote! { + tokens_helper(v, &#name.#spans) + }) + } + Type::Group(_) => { + let name = name.tokens(); + Some(quote! { + tokens_helper(v, &#name.span) + }) + } + Type::Syn(t) => { + fn requires_full(features: &Features) -> bool { + features.any.contains("full") && features.any.len() == 1 + } + let mut res = simple_visit(t, name); + let target = defs.types.iter().find(|ty| ty.ident == *t).unwrap(); + if requires_full(&target.features) && !requires_full(features) { + res = quote!(full!(#res)); + } + Some(res) + } + Type::Ext(t) if gen::TERMINAL_TYPES.contains(&&t[..]) => Some(simple_visit(t, name)), + Type::Ext(_) | Type::Std(_) => None, + } +} + +fn node(traits: &mut TokenStream, impls: &mut TokenStream, s: &Node, defs: &Definitions) { + let under_name = gen::under_name(&s.ident); + let ty = Ident::new(&s.ident, Span::call_site()); + let visit_fn = Ident::new(&format!("visit_{}", under_name), Span::call_site()); + + let mut visit_impl = TokenStream::new(); + + match &s.data { + Data::Enum(variants) => { + let mut visit_variants = TokenStream::new(); + + for (variant, fields) in variants { + let variant_ident = Ident::new(variant, Span::call_site()); + + if fields.is_empty() { + visit_variants.extend(quote! { + #ty::#variant_ident => {} + }); + } else { + let mut bind_visit_fields = TokenStream::new(); + let mut visit_fields = TokenStream::new(); + + for (idx, ty) in fields.iter().enumerate() { + let name = format!("_binding_{}", idx); + let binding = Ident::new(&name, Span::call_site()); + + bind_visit_fields.extend(quote! { + #binding, + }); + + let borrowed_binding = Borrowed(quote!(#binding)); + + visit_fields.extend( + visit(ty, &s.features, defs, &borrowed_binding) + .unwrap_or_else(|| noop_visit(&borrowed_binding)), + ); + + visit_fields.extend(quote!(;)); + } + + visit_variants.extend(quote! { + #ty::#variant_ident(#bind_visit_fields) => { + #visit_fields + } + }); + } + } + + let nonexhaustive = if s.exhaustive { + None + } else { + Some(quote!(_ => unreachable!())) + }; + + visit_impl.extend(quote! { + match node { + #visit_variants + #nonexhaustive + } + }); + } + Data::Struct(fields) => { + for (field, ty) in fields { + if let Type::Syn(ty) = ty { + if ty == "Reserved" { + continue; + } + } + + let id = Ident::new(&field, Span::call_site()); + let ref_toks = Owned(quote!(node.#id)); + let visit_field = visit(&ty, &s.features, defs, &ref_toks) + .unwrap_or_else(|| noop_visit(&ref_toks)); + visit_impl.extend(quote! { + #visit_field; + }); + } + } + Data::Private => { + if ty == "Ident" { + visit_impl.extend(quote! { + v.visit_span(&node.span()); + }); + } + } + } + + let ast_lifetime = if s.ident == "Span" { + None + } else { + Some(quote!('ast)) + }; + + traits.extend(quote! { + fn #visit_fn(&mut self, i: &#ast_lifetime #ty) { + #visit_fn(self, i) + } + }); + + impls.extend(quote! { + pub fn #visit_fn<'ast, V>(v: &mut V, node: &#ast_lifetime #ty) + where + V: Visit<'ast> + ?Sized, + { + #visit_impl + } + }); +} + +pub fn generate(defs: &Definitions) -> Result<()> { + let (traits, impls) = gen::traverse(defs, node); + let full_macro = full::get_macro(); + file::write( + VISIT_SRC, + quote! { + #![allow(unused_variables)] + + use crate::*; + #[cfg(any(feature = "full", feature = "derive"))] + use crate::punctuated::Punctuated; + use proc_macro2::Span; + #[cfg(any(feature = "full", feature = "derive"))] + use crate::gen::helper::visit::*; + + #full_macro + + #[cfg(any(feature = "full", feature = "derive"))] + macro_rules! skip { + ($($tt:tt)*) => {}; + } + + /// Syntax tree traversal to walk a shared borrow of a syntax tree. + /// + /// See the [module documentation] for details. + /// + /// [module documentation]: self + /// + /// *This trait is available if Syn is built with the `"visit"` feature.* + pub trait Visit<'ast> { + #traits + } + + #impls + }, + )?; + Ok(()) +} diff --git a/syn/codegen/src/visit_mut.rs b/syn/codegen/src/visit_mut.rs new file mode 100644 index 0000000..71e56b3 --- /dev/null +++ b/syn/codegen/src/visit_mut.rs @@ -0,0 +1,262 @@ +use crate::operand::{Borrowed, Operand, Owned}; +use crate::{file, full, gen}; +use anyhow::Result; +use proc_macro2::{Ident, Span, TokenStream}; +use quote::quote; +use syn::Index; +use syn_codegen::{Data, Definitions, Features, Node, Type}; + +const VISIT_MUT_SRC: &str = "../src/gen/visit_mut.rs"; + +fn simple_visit(item: &str, name: &Operand) -> TokenStream { + let ident = gen::under_name(item); + let method = Ident::new(&format!("visit_{}_mut", ident), Span::call_site()); + let name = name.ref_mut_tokens(); + quote! { + v.#method(#name) + } +} + +fn noop_visit(name: &Operand) -> TokenStream { + let name = name.tokens(); + quote! { + skip!(#name) + } +} + +fn visit( + ty: &Type, + features: &Features, + defs: &Definitions, + name: &Operand, +) -> Option { + match ty { + Type::Box(t) => { + let name = name.owned_tokens(); + visit(t, features, defs, &Owned(quote!(*#name))) + } + Type::Vec(t) => { + let operand = Borrowed(quote!(it)); + let val = visit(t, features, defs, &operand)?; + let name = name.ref_mut_tokens(); + Some(quote! { + for it in #name { + #val + } + }) + } + Type::Punctuated(p) => { + let operand = Borrowed(quote!(it)); + let val = visit(&p.element, features, defs, &operand)?; + let name = name.ref_mut_tokens(); + Some(quote! { + for el in Punctuated::pairs_mut(#name) { + let (it, p) = el.into_tuple(); + #val; + if let Some(p) = p { + tokens_helper(v, &mut p.spans); + } + } + }) + } + Type::Option(t) => { + let it = Borrowed(quote!(it)); + let val = visit(t, features, defs, &it)?; + let name = name.owned_tokens(); + Some(quote! { + if let Some(it) = &mut #name { + #val + } + }) + } + Type::Tuple(t) => { + let mut code = TokenStream::new(); + for (i, elem) in t.iter().enumerate() { + let name = name.tokens(); + let i = Index::from(i); + let it = Owned(quote!((#name).#i)); + let val = visit(elem, features, defs, &it).unwrap_or_else(|| noop_visit(&it)); + code.extend(val); + code.extend(quote!(;)); + } + Some(code) + } + Type::Token(t) => { + let name = name.tokens(); + let repr = &defs.tokens[t]; + let is_keyword = repr.chars().next().unwrap().is_alphabetic(); + let spans = if is_keyword { + quote!(span) + } else { + quote!(spans) + }; + Some(quote! { + tokens_helper(v, &mut #name.#spans) + }) + } + Type::Group(_) => { + let name = name.tokens(); + Some(quote! { + tokens_helper(v, &mut #name.span) + }) + } + Type::Syn(t) => { + fn requires_full(features: &Features) -> bool { + features.any.contains("full") && features.any.len() == 1 + } + let mut res = simple_visit(t, name); + let target = defs.types.iter().find(|ty| ty.ident == *t).unwrap(); + if requires_full(&target.features) && !requires_full(features) { + res = quote!(full!(#res)); + } + Some(res) + } + Type::Ext(t) if gen::TERMINAL_TYPES.contains(&&t[..]) => Some(simple_visit(t, name)), + Type::Ext(_) | Type::Std(_) => None, + } +} + +fn node(traits: &mut TokenStream, impls: &mut TokenStream, s: &Node, defs: &Definitions) { + let under_name = gen::under_name(&s.ident); + let ty = Ident::new(&s.ident, Span::call_site()); + let visit_mut_fn = Ident::new(&format!("visit_{}_mut", under_name), Span::call_site()); + + let mut visit_mut_impl = TokenStream::new(); + + match &s.data { + Data::Enum(variants) => { + let mut visit_mut_variants = TokenStream::new(); + + for (variant, fields) in variants { + let variant_ident = Ident::new(variant, Span::call_site()); + + if fields.is_empty() { + visit_mut_variants.extend(quote! { + #ty::#variant_ident => {} + }); + } else { + let mut bind_visit_mut_fields = TokenStream::new(); + let mut visit_mut_fields = TokenStream::new(); + + for (idx, ty) in fields.iter().enumerate() { + let name = format!("_binding_{}", idx); + let binding = Ident::new(&name, Span::call_site()); + + bind_visit_mut_fields.extend(quote! { + #binding, + }); + + let borrowed_binding = Borrowed(quote!(#binding)); + + visit_mut_fields.extend( + visit(ty, &s.features, defs, &borrowed_binding) + .unwrap_or_else(|| noop_visit(&borrowed_binding)), + ); + + visit_mut_fields.extend(quote!(;)); + } + + visit_mut_variants.extend(quote! { + #ty::#variant_ident(#bind_visit_mut_fields) => { + #visit_mut_fields + } + }); + } + } + + let nonexhaustive = if s.exhaustive { + None + } else { + Some(quote!(_ => unreachable!())) + }; + + visit_mut_impl.extend(quote! { + match node { + #visit_mut_variants + #nonexhaustive + } + }); + } + Data::Struct(fields) => { + for (field, ty) in fields { + if let Type::Syn(ty) = ty { + if ty == "Reserved" { + continue; + } + } + + let id = Ident::new(&field, Span::call_site()); + let ref_toks = Owned(quote!(node.#id)); + let visit_mut_field = visit(&ty, &s.features, defs, &ref_toks) + .unwrap_or_else(|| noop_visit(&ref_toks)); + visit_mut_impl.extend(quote! { + #visit_mut_field; + }); + } + } + Data::Private => { + if ty == "Ident" { + visit_mut_impl.extend(quote! { + let mut span = node.span(); + v.visit_span_mut(&mut span); + node.set_span(span); + }); + } + } + } + + traits.extend(quote! { + fn #visit_mut_fn(&mut self, i: &mut #ty) { + #visit_mut_fn(self, i) + } + }); + + impls.extend(quote! { + pub fn #visit_mut_fn(v: &mut V, node: &mut #ty) + where + V: VisitMut + ?Sized, + { + #visit_mut_impl + } + }); +} + +pub fn generate(defs: &Definitions) -> Result<()> { + let (traits, impls) = gen::traverse(defs, node); + let full_macro = full::get_macro(); + file::write( + VISIT_MUT_SRC, + quote! { + #![allow(unused_variables)] + + use crate::*; + #[cfg(any(feature = "full", feature = "derive"))] + use crate::punctuated::Punctuated; + use proc_macro2::Span; + #[cfg(any(feature = "full", feature = "derive"))] + use crate::gen::helper::visit_mut::*; + + #full_macro + + #[cfg(any(feature = "full", feature = "derive"))] + macro_rules! skip { + ($($tt:tt)*) => {}; + } + + /// Syntax tree traversal to mutate an exclusive borrow of a syntax tree in + /// place. + /// + /// See the [module documentation] for details. + /// + /// [module documentation]: self + /// + /// *This trait is available if Syn is built with the `"visit-mut"` feature.* + pub trait VisitMut { + #traits + } + + #impls + }, + )?; + Ok(()) +} diff --git a/syn/dev/Cargo.toml b/syn/dev/Cargo.toml new file mode 100644 index 0000000..79486c1 --- /dev/null +++ b/syn/dev/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "syn-dev" +version = "0.0.0" +authors = ["David Tolnay "] +edition = "2018" +publish = false + +[lib] +path = "parse.rs" +proc-macro = true + +[[bin]] +path = "main.rs" +name = "syn-dev" + +[dependencies] +quote = "1.0" + +[dependencies.syn] +path = ".." +default-features = false +features = ["parsing", "full", "extra-traits", "proc-macro"] diff --git a/syn/dev/README.md b/syn/dev/README.md new file mode 100644 index 0000000..91b9846 --- /dev/null +++ b/syn/dev/README.md @@ -0,0 +1,6 @@ +A little project skeleton for troubleshooting Syn's parsers during development, +especially when adding support for new Rust syntax. + +Place a sample of the syntax you are working on into main.rs and then run `cargo +check` to try parsing it, revealing the resulting syntax tree or else showing +the position and error message if the input fails to parse. diff --git a/syn/dev/main.rs b/syn/dev/main.rs new file mode 100644 index 0000000..eb67546 --- /dev/null +++ b/syn/dev/main.rs @@ -0,0 +1,4 @@ +syn_dev::r#mod! { + // Write Rust code here and run `cargo check` to have Syn parse it. + +} diff --git a/syn/dev/parse.rs b/syn/dev/parse.rs new file mode 100644 index 0000000..2a92550 --- /dev/null +++ b/syn/dev/parse.rs @@ -0,0 +1,18 @@ +extern crate proc_macro; + +use proc_macro::TokenStream; +use quote::quote; +use syn::File; + +#[proc_macro] +pub fn r#mod(input: TokenStream) -> TokenStream { + let compile_error = syn::parse::(input) + .map(|file| println!("{:#?}", file)) + .map_err(|err| err.to_compile_error()) + .err(); + + TokenStream::from(quote! { + #compile_error + fn main() {} + }) +} diff --git a/syn/examples/README.md b/syn/examples/README.md new file mode 100644 index 0000000..fdd69d6 --- /dev/null +++ b/syn/examples/README.md @@ -0,0 +1,19 @@ +### [`dump-syntax`](dump-syntax) + +Little utility to parse a Rust source file into a `syn::File` and print out a +debug representation of the syntax tree. + +### [`heapsize`](heapsize) + +An example implementation of a derive macro that generates trait impls. + +### [`lazy-static`](lazy-static) + +An example of parsing a custom syntax within a `functionlike!(...)` procedural +macro. Demonstrates how to trigger custom warnings and error messages on +individual tokens of the input. + +### [`trace-var`](trace-var) + +An attribute procedural macro that uses a syntax tree traversal to transform +certain syntax tree nodes in a function body. diff --git a/syn/examples/dump-syntax/Cargo.toml b/syn/examples/dump-syntax/Cargo.toml new file mode 100644 index 0000000..0bc9f62 --- /dev/null +++ b/syn/examples/dump-syntax/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "dump-syntax" +version = "0.0.0" +authors = ["David Tolnay "] +edition = "2018" +publish = false + +[dependencies] +colored = "1.7" +proc-macro2 = { version = "1.0", features = ["span-locations"] } + +[dependencies.syn] +path = "../.." +default-features = false +features = ["parsing", "full", "extra-traits"] + +[workspace] diff --git a/syn/examples/dump-syntax/README.md b/syn/examples/dump-syntax/README.md new file mode 100644 index 0000000..37c84d8 --- /dev/null +++ b/syn/examples/dump-syntax/README.md @@ -0,0 +1,28 @@ +Parse a Rust source file into a `syn::File` and print out a debug representation +of the syntax tree. + +Use the following command from this directory to test this program by running it +on its own source code: + +``` +cargo run -- src/main.rs +``` + +The output will begin with: + +``` +File { + shebang: None, + attrs: [ + Attribute { + pound_token: Pound, + style: Inner( + Bang + ), + bracket_token: Bracket, + path: Path { + leading_colon: None, + segments: [ + ... +} +``` diff --git a/syn/examples/dump-syntax/src/main.rs b/syn/examples/dump-syntax/src/main.rs new file mode 100644 index 0000000..240b7a2 --- /dev/null +++ b/syn/examples/dump-syntax/src/main.rs @@ -0,0 +1,149 @@ +//! Parse a Rust source file into a `syn::File` and print out a debug +//! representation of the syntax tree. +//! +//! Use the following command from this directory to test this program by +//! running it on its own source code: +//! +//! cargo run -- src/main.rs +//! +//! The output will begin with: +//! +//! File { +//! shebang: None, +//! attrs: [ +//! Attribute { +//! pound_token: Pound, +//! style: Inner( +//! ... +//! } + +use std::borrow::Cow; +use std::env; +use std::ffi::OsStr; +use std::fmt::{self, Display}; +use std::fs; +use std::io::{self, Write}; +use std::path::{Path, PathBuf}; +use std::process; + +use colored::Colorize; + +enum Error { + IncorrectUsage, + ReadFile(io::Error), + ParseFile { + error: syn::Error, + filepath: PathBuf, + source_code: String, + }, +} + +impl Display for Error { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + use self::Error::*; + + match self { + IncorrectUsage => write!(f, "Usage: dump-syntax path/to/filename.rs"), + ReadFile(error) => write!(f, "Unable to read file: {}", error), + ParseFile { + error, + filepath, + source_code, + } => render_location(f, error, filepath, source_code), + } + } +} + +fn main() { + if let Err(error) = try_main() { + let _ = writeln!(io::stderr(), "{}", error); + process::exit(1); + } +} + +fn try_main() -> Result<(), Error> { + let mut args = env::args_os(); + let _ = args.next(); // executable name + + let filepath = match (args.next(), args.next()) { + (Some(arg), None) => PathBuf::from(arg), + _ => return Err(Error::IncorrectUsage), + }; + + let code = fs::read_to_string(&filepath).map_err(Error::ReadFile)?; + let syntax = syn::parse_file(&code).map_err({ + |error| Error::ParseFile { + error, + filepath, + source_code: code, + } + })?; + println!("{:#?}", syntax); + + Ok(()) +} + +// Render a rustc-style error message, including colors. +// +// error: Syn unable to parse file +// --> main.rs:40:17 +// | +// 40 | fn fmt(&self formatter: &mut fmt::Formatter) -> fmt::Result { +// | ^^^^^^^^^ expected `,` +// +fn render_location( + formatter: &mut fmt::Formatter, + err: &syn::Error, + filepath: &Path, + code: &str, +) -> fmt::Result { + let start = err.span().start(); + let mut end = err.span().end(); + + if start.line == end.line && start.column == end.column { + return render_fallback(formatter, err); + } + + let code_line = match code.lines().nth(start.line - 1) { + Some(line) => line, + None => return render_fallback(formatter, err), + }; + + if end.line > start.line { + end.line = start.line; + end.column = code_line.len(); + } + + let filename = filepath + .file_name() + .map(OsStr::to_string_lossy) + .unwrap_or(Cow::Borrowed("main.rs")); + + write!( + formatter, + "\n\ + {error}{header}\n\ + {indent}{arrow} {filename}:{linenum}:{colnum}\n\ + {indent} {pipe}\n\ + {label} {pipe} {code}\n\ + {indent} {pipe} {offset}{underline} {message}\n\ + ", + error = "error".red().bold(), + header = ": Syn unable to parse file".bold(), + indent = " ".repeat(start.line.to_string().len()), + arrow = "-->".blue().bold(), + filename = filename, + linenum = start.line, + colnum = start.column, + pipe = "|".blue().bold(), + label = start.line.to_string().blue().bold(), + code = code_line.trim_end(), + offset = " ".repeat(start.column), + underline = "^".repeat(end.column - start.column).red().bold(), + message = err.to_string().red(), + ) +} + +fn render_fallback(formatter: &mut fmt::Formatter, err: &syn::Error) -> fmt::Result { + write!(formatter, "Unable to parse file: {}", err) +} diff --git a/syn/examples/heapsize/Cargo.toml b/syn/examples/heapsize/Cargo.toml new file mode 100644 index 0000000..9b19214 --- /dev/null +++ b/syn/examples/heapsize/Cargo.toml @@ -0,0 +1,2 @@ +[workspace] +members = ["example", "heapsize", "heapsize_derive"] diff --git a/syn/examples/heapsize/README.md b/syn/examples/heapsize/README.md new file mode 100644 index 0000000..e789559 --- /dev/null +++ b/syn/examples/heapsize/README.md @@ -0,0 +1,72 @@ +A derive macro that generates trait impls. + +- [`heapsize/src/lib.rs`](heapsize/src/lib.rs) +- [`heapsize_derive/src/lib.rs`](heapsize_derive/src/lib.rs) +- [`example/src/main.rs`](example/src/main.rs) + +We are deriving the `HeapSize` trait which computes an estimate of the amount of +heap memory owned by a value. + +```rust +pub trait HeapSize { + /// Total number of bytes of heap memory owned by `self`. + fn heap_size_of_children(&self) -> usize; +} +``` + +The derive macro allows users to write `#[derive(HeapSize)]` on data structures +in their program. + +```rust +#[derive(HeapSize)] +struct Demo<'a, T: ?Sized> { + a: Box, + b: u8, + c: &'a str, + d: String, +} +``` + +The trait impl generated by the derive macro here would look like: + +```rust +impl<'a, T: ?Sized + heapsize::HeapSize> heapsize::HeapSize for Demo<'a, T> { + fn heap_size_of_children(&self) -> usize { + 0 + heapsize::HeapSize::heap_size_of_children(&self.a) + + heapsize::HeapSize::heap_size_of_children(&self.b) + + heapsize::HeapSize::heap_size_of_children(&self.c) + + heapsize::HeapSize::heap_size_of_children(&self.d) + } +} +``` + +The implementation of `heapsize_derive` demonstrates some attention to "spans" +of error messages. For each subexpression in the generated code we apply the +span of the input fragment under which we would want to trigger a compiler error +if the subexpression fails to compile. In this example, each recursive call to +`heap_size_of_children` is associated with the span of the corresponding struct +field. Thus we get errors in the right place if any of the field types do not +implement the `HeapSize` trait. + +``` +error[E0277]: the trait bound `std::thread::Thread: HeapSize` is not satisfied + --> src/main.rs:7:5 + | +7 | bad: std::thread::Thread, + | ^^^ the trait `HeapSize` is not implemented for `std::thread::Thread` +``` + +Some unstable APIs in the `proc-macro2` crate let us improve this further by +joining together the span of the field name and the field type. There is no +difference in our code — everything is as shown in this directory — +but building the example crate with `cargo build` shows errors like the one +above and building with `RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo build` +is able to show errors like the following. + +``` +error[E0277]: the trait bound `std::thread::Thread: HeapSize` is not satisfied + --> src/main.rs:7:5 + | +7 | bad: std::thread::Thread, + | ^^^^^^^^^^^^^^^^^^^^^^^^ the trait `HeapSize` is not implemented for `std::thread::Thread` +``` diff --git a/syn/examples/heapsize/example/Cargo.toml b/syn/examples/heapsize/example/Cargo.toml new file mode 100644 index 0000000..85c7699 --- /dev/null +++ b/syn/examples/heapsize/example/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "heapsize_example" +version = "0.0.0" +authors = ["David Tolnay "] +edition = "2018" +publish = false + +[dependencies] +heapsize = { path = "../heapsize" } diff --git a/syn/examples/heapsize/example/src/main.rs b/syn/examples/heapsize/example/src/main.rs new file mode 100644 index 0000000..9332b11 --- /dev/null +++ b/syn/examples/heapsize/example/src/main.rs @@ -0,0 +1,28 @@ +use heapsize::HeapSize; + +#[derive(HeapSize)] +struct Demo<'a, T: ?Sized> { + a: Box, + b: u8, + c: &'a str, + d: String, +} + +fn main() { + let demo = Demo { + a: b"bytestring".to_vec().into_boxed_slice(), + b: 255, + c: "&'static str", + d: "String".to_owned(), + }; + + // 10 + 0 + 0 + 6 = 16 + println!( + "heap size = {} + {} + {} + {} = {}", + demo.a.heap_size_of_children(), + demo.b.heap_size_of_children(), + demo.c.heap_size_of_children(), + demo.d.heap_size_of_children(), + demo.heap_size_of_children() + ); +} diff --git a/syn/examples/heapsize/heapsize/Cargo.toml b/syn/examples/heapsize/heapsize/Cargo.toml new file mode 100644 index 0000000..27bb954 --- /dev/null +++ b/syn/examples/heapsize/heapsize/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "heapsize" +version = "0.0.0" +authors = ["David Tolnay "] +edition = "2018" +publish = false + +[dependencies] +heapsize_derive = { path = "../heapsize_derive" } diff --git a/syn/examples/heapsize/heapsize/src/lib.rs b/syn/examples/heapsize/heapsize/src/lib.rs new file mode 100644 index 0000000..30bb6d6 --- /dev/null +++ b/syn/examples/heapsize/heapsize/src/lib.rs @@ -0,0 +1,64 @@ +use std::mem; + +pub use heapsize_derive::*; + +pub trait HeapSize { + /// Total number of bytes of heap memory owned by `self`. + /// + /// Does not include the size of `self` itself, which may or may not be on + /// the heap. Includes only children of `self`, meaning things pointed to by + /// `self`. + fn heap_size_of_children(&self) -> usize; +} + +// +// In a real version of this library there would be lots more impls here, but +// here are some interesting ones. +// + +impl HeapSize for u8 { + /// A `u8` does not own any heap memory. + fn heap_size_of_children(&self) -> usize { + 0 + } +} + +impl HeapSize for String { + /// A `String` owns enough heap memory to hold its reserved capacity. + fn heap_size_of_children(&self) -> usize { + self.capacity() + } +} + +impl HeapSize for Box +where + T: ?Sized + HeapSize, +{ + /// A `Box` owns however much heap memory was allocated to hold the value of + /// type `T` that we placed on the heap, plus transitively however much `T` + /// itself owns. + fn heap_size_of_children(&self) -> usize { + mem::size_of_val(&**self) + (**self).heap_size_of_children() + } +} + +impl HeapSize for [T] +where + T: HeapSize, +{ + /// Sum of heap memory owned by each element of a dynamically sized slice of + /// `T`. + fn heap_size_of_children(&self) -> usize { + self.iter().map(HeapSize::heap_size_of_children).sum() + } +} + +impl<'a, T> HeapSize for &'a T +where + T: ?Sized, +{ + /// A shared reference does not own heap memory. + fn heap_size_of_children(&self) -> usize { + 0 + } +} diff --git a/syn/examples/heapsize/heapsize_derive/Cargo.toml b/syn/examples/heapsize/heapsize_derive/Cargo.toml new file mode 100644 index 0000000..f4357b9 --- /dev/null +++ b/syn/examples/heapsize/heapsize_derive/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "heapsize_derive" +version = "0.0.0" +authors = ["David Tolnay "] +edition = "2018" +publish = false + +[lib] +proc-macro = true + +[dependencies] +proc-macro2 = "1.0" +quote = "1.0" +syn = { path = "../../.." } diff --git a/syn/examples/heapsize/heapsize_derive/src/lib.rs b/syn/examples/heapsize/heapsize_derive/src/lib.rs new file mode 100644 index 0000000..9176b29 --- /dev/null +++ b/syn/examples/heapsize/heapsize_derive/src/lib.rs @@ -0,0 +1,96 @@ +extern crate proc_macro; + +use proc_macro2::TokenStream; +use quote::{quote, quote_spanned}; +use syn::spanned::Spanned; +use syn::{parse_macro_input, parse_quote, Data, DeriveInput, Fields, GenericParam, Generics, Index}; + +#[proc_macro_derive(HeapSize)] +pub fn derive_heap_size(input: proc_macro::TokenStream) -> proc_macro::TokenStream { + // Parse the input tokens into a syntax tree. + let input = parse_macro_input!(input as DeriveInput); + + // Used in the quasi-quotation below as `#name`. + let name = input.ident; + + // Add a bound `T: HeapSize` to every type parameter T. + let generics = add_trait_bounds(input.generics); + let (impl_generics, ty_generics, where_clause) = generics.split_for_impl(); + + // Generate an expression to sum up the heap size of each field. + let sum = heap_size_sum(&input.data); + + let expanded = quote! { + // The generated impl. + impl #impl_generics heapsize::HeapSize for #name #ty_generics #where_clause { + fn heap_size_of_children(&self) -> usize { + #sum + } + } + }; + + // Hand the output tokens back to the compiler. + proc_macro::TokenStream::from(expanded) +} + +// Add a bound `T: HeapSize` to every type parameter T. +fn add_trait_bounds(mut generics: Generics) -> Generics { + for param in &mut generics.params { + if let GenericParam::Type(ref mut type_param) = *param { + type_param.bounds.push(parse_quote!(heapsize::HeapSize)); + } + } + generics +} + +// Generate an expression to sum up the heap size of each field. +fn heap_size_sum(data: &Data) -> TokenStream { + match *data { + Data::Struct(ref data) => { + match data.fields { + Fields::Named(ref fields) => { + // Expands to an expression like + // + // 0 + self.x.heap_size() + self.y.heap_size() + self.z.heap_size() + // + // but using fully qualified function call syntax. + // + // We take some care to use the span of each `syn::Field` as + // the span of the corresponding `heap_size_of_children` + // call. This way if one of the field types does not + // implement `HeapSize` then the compiler's error message + // underlines which field it is. An example is shown in the + // readme of the parent directory. + let recurse = fields.named.iter().map(|f| { + let name = &f.ident; + quote_spanned! {f.span()=> + heapsize::HeapSize::heap_size_of_children(&self.#name) + } + }); + quote! { + 0 #(+ #recurse)* + } + } + Fields::Unnamed(ref fields) => { + // Expands to an expression like + // + // 0 + self.0.heap_size() + self.1.heap_size() + self.2.heap_size() + let recurse = fields.unnamed.iter().enumerate().map(|(i, f)| { + let index = Index::from(i); + quote_spanned! {f.span()=> + heapsize::HeapSize::heap_size_of_children(&self.#index) + } + }); + quote! { + 0 #(+ #recurse)* + } + } + Fields::Unit => { + // Unit structs cannot own more than 0 bytes of heap memory. + quote!(0) + } + } + } + Data::Enum(_) | Data::Union(_) => unimplemented!(), + } +} diff --git a/syn/examples/lazy-static/Cargo.toml b/syn/examples/lazy-static/Cargo.toml new file mode 100644 index 0000000..586e547 --- /dev/null +++ b/syn/examples/lazy-static/Cargo.toml @@ -0,0 +1,2 @@ +[workspace] +members = ["example", "lazy-static"] diff --git a/syn/examples/lazy-static/README.md b/syn/examples/lazy-static/README.md new file mode 100644 index 0000000..bc64585 --- /dev/null +++ b/syn/examples/lazy-static/README.md @@ -0,0 +1,42 @@ +An example of parsing a custom syntax within a `functionlike!(...)` procedural +macro. Demonstrates how to trigger custom warnings and error messages on +individual tokens of the input. + +- [`lazy-static/src/lib.rs`](lazy-static/src/lib.rs) +- [`example/src/main.rs`](example/src/main.rs) + +The library implements a `lazy_static!` macro similar to the one from the real +[`lazy_static`](https://docs.rs/lazy_static/1.0.0/lazy_static/) crate on +crates.io. + +```rust +lazy_static! { + static ref USERNAME: Regex = Regex::new("^[a-z0-9_-]{3,16}$").unwrap(); +} +``` + +Compile and run the example by doing `cargo run` in the directory of the +`example` crate. + +The implementation shows how to trigger custom warnings and error messages on +the macro input. For example if you try adding an uncreatively named `FOO` lazy +static, the macro will scold you with the following warning. + +``` +warning: come on, pick a more creative name + --> src/main.rs:10:16 + | +10 | static ref FOO: String = "lazy_static".to_owned(); + | ^^^ +``` + +And if you try to lazily initialize `() = ()`, the macro will outright refuse to +compile it for you. + +``` +error: I can't think of a legitimate use for lazily initializing the value `()` + --> src/main.rs:10:27 + | +10 | static ref UNIT: () = (); + | ^^ +``` diff --git a/syn/examples/lazy-static/example/Cargo.toml b/syn/examples/lazy-static/example/Cargo.toml new file mode 100644 index 0000000..716b08c --- /dev/null +++ b/syn/examples/lazy-static/example/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "example" +version = "0.0.0" +authors = ["David Tolnay "] +edition = "2018" +publish = false + +[dependencies] +lazy_static = { path = "../lazy-static" } +regex = "0.2" diff --git a/syn/examples/lazy-static/example/src/main.rs b/syn/examples/lazy-static/example/src/main.rs new file mode 100644 index 0000000..c4f64af --- /dev/null +++ b/syn/examples/lazy-static/example/src/main.rs @@ -0,0 +1,20 @@ +use lazy_static::lazy_static; +use regex::Regex; + +lazy_static! { + static ref USERNAME: Regex = { + println!("Compiling username regex..."); + Regex::new("^[a-z0-9_-]{3,16}$").unwrap() + }; +} + +fn main() { + println!("Let's validate some usernames."); + validate("fergie"); + validate("will.i.am"); +} + +fn validate(name: &str) { + // The USERNAME regex is compiled lazily the first time its value is accessed. + println!("is_match({:?}): {}", name, USERNAME.is_match(name)); +} diff --git a/syn/examples/lazy-static/lazy-static/Cargo.toml b/syn/examples/lazy-static/lazy-static/Cargo.toml new file mode 100644 index 0000000..bf65787 --- /dev/null +++ b/syn/examples/lazy-static/lazy-static/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "lazy_static" +version = "0.0.0" +authors = ["David Tolnay "] +edition = "2018" +publish = false + +[lib] +proc-macro = true + +[dependencies] +proc-macro2 = { version = "1.0", features = ["nightly"] } +quote = "1.0" +syn = { path = "../../../", features = ["full"] } diff --git a/syn/examples/lazy-static/lazy-static/src/lib.rs b/syn/examples/lazy-static/lazy-static/src/lib.rs new file mode 100644 index 0000000..254ca72 --- /dev/null +++ b/syn/examples/lazy-static/lazy-static/src/lib.rs @@ -0,0 +1,143 @@ +#![recursion_limit = "128"] +#![feature(proc_macro_diagnostic)] + +extern crate proc_macro; +use self::proc_macro::TokenStream; + +use quote::{quote, quote_spanned}; +use syn::parse::{Parse, ParseStream, Result}; +use syn::spanned::Spanned; +use syn::{parse_macro_input, Expr, Ident, Token, Type, Visibility}; + +/// Parses the following syntax, which aligns with the input of the real +/// `lazy_static` crate. +/// +/// lazy_static! { +/// $VISIBILITY static ref $NAME: $TYPE = $EXPR; +/// } +/// +/// For example: +/// +/// lazy_static! { +/// static ref USERNAME: Regex = Regex::new("^[a-z0-9_-]{3,16}$").unwrap(); +/// } +struct LazyStatic { + visibility: Visibility, + name: Ident, + ty: Type, + init: Expr, +} + +impl Parse for LazyStatic { + fn parse(input: ParseStream) -> Result { + let visibility: Visibility = input.parse()?; + input.parse::()?; + input.parse::()?; + let name: Ident = input.parse()?; + input.parse::()?; + let ty: Type = input.parse()?; + input.parse::()?; + let init: Expr = input.parse()?; + input.parse::()?; + Ok(LazyStatic { + visibility, + name, + ty, + init, + }) + } +} + +#[proc_macro] +pub fn lazy_static(input: TokenStream) -> TokenStream { + let LazyStatic { + visibility, + name, + ty, + init, + } = parse_macro_input!(input as LazyStatic); + + // The warning looks like this. + // + // warning: come on, pick a more creative name + // --> src/main.rs:10:16 + // | + // 10 | static ref FOO: String = "lazy_static".to_owned(); + // | ^^^ + if name == "FOO" { + name.span() + .unwrap() + .warning("come on, pick a more creative name") + .emit(); + } + + // The error looks like this. + // + // error: I can't think of a legitimate use for lazily initializing the value `()` + // --> src/main.rs:10:27 + // | + // 10 | static ref UNIT: () = (); + // | ^^ + if let Expr::Tuple(ref init) = init { + if init.elems.is_empty() { + init.span() + .unwrap() + .error("I can't think of a legitimate use for lazily initializing the value `()`") + .emit(); + return TokenStream::new(); + } + } + + // Assert that the static type implements Sync. If not, user sees an error + // message like the following. We span this assertion with the field type's + // line/column so that the error message appears in the correct place. + // + // error[E0277]: the trait bound `*const (): std::marker::Sync` is not satisfied + // --> src/main.rs:10:21 + // | + // 10 | static ref PTR: *const () = &(); + // | ^^^^^^^^^ `*const ()` cannot be shared between threads safely + let assert_sync = quote_spanned! {ty.span()=> + struct _AssertSync where #ty: std::marker::Sync; + }; + + // Check for Sized. Not vital to check here, but the error message is less + // confusing this way than if they get a Sized error in one of our + // implementation details where it assumes Sized. + // + // error[E0277]: the trait bound `str: std::marker::Sized` is not satisfied + // --> src/main.rs:10:19 + // | + // 10 | static ref A: str = ""; + // | ^^^ `str` does not have a constant size known at compile-time + let assert_sized = quote_spanned! {ty.span()=> + struct _AssertSized where #ty: std::marker::Sized; + }; + + let init_ptr = quote_spanned! {init.span()=> + Box::into_raw(Box::new(#init)) + }; + + let expanded = quote! { + #visibility struct #name; + + impl std::ops::Deref for #name { + type Target = #ty; + + fn deref(&self) -> &#ty { + #assert_sync + #assert_sized + + static ONCE: std::sync::Once = std::sync::Once::new(); + static mut VALUE: *mut #ty = 0 as *mut #ty; + + unsafe { + ONCE.call_once(|| VALUE = #init_ptr); + &*VALUE + } + } + } + }; + + TokenStream::from(expanded) +} diff --git a/syn/examples/trace-var/Cargo.toml b/syn/examples/trace-var/Cargo.toml new file mode 100644 index 0000000..b54454d --- /dev/null +++ b/syn/examples/trace-var/Cargo.toml @@ -0,0 +1,2 @@ +[workspace] +members = ["example", "trace-var"] diff --git a/syn/examples/trace-var/README.md b/syn/examples/trace-var/README.md new file mode 100644 index 0000000..b93fae2 --- /dev/null +++ b/syn/examples/trace-var/README.md @@ -0,0 +1,61 @@ +An example of an attribute procedural macro. The `#[trace_var(...)]` attribute +prints the value of the given variables each time they are reassigned. + +- [`trace-var/src/lib.rs`](trace-var/src/lib.rs) +- [`example/src/main.rs`](example/src/main.rs) + +Consider the following factorial implementation. + +```rust +#[trace_var(p, n)] +fn factorial(mut n: u64) -> u64 { + let mut p = 1; + while n > 1 { + p *= n; + n -= 1; + } + p +} +``` + +Invoking this with `factorial(8)` prints all the values of `p` and `n` during +the execution of the function. + +``` +p = 1 +p = 8 +n = 7 +p = 56 +n = 6 +p = 336 +n = 5 +p = 1680 +n = 4 +p = 6720 +n = 3 +p = 20160 +n = 2 +p = 40320 +n = 1 +``` + +The procedural macro uses a syntax tree [`Fold`] to rewrite every `let` +statement and assignment expression in the following way: + +[`Fold`]: https://docs.rs/syn/1.0/syn/fold/trait.Fold.html + +```rust +// Before +let VAR = INIT; + +// After +let VAR = { let VAR = INIT; println!("VAR = {:?}", VAR); VAR }; +``` + +```rust +// Before +VAR = INIT + +// After +{ VAR = INIT; println!("VAR = {:?}", VAR); } +``` diff --git a/syn/examples/trace-var/example/Cargo.toml b/syn/examples/trace-var/example/Cargo.toml new file mode 100644 index 0000000..d2ad650 --- /dev/null +++ b/syn/examples/trace-var/example/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "example" +version = "0.0.0" +authors = ["David Tolnay "] +edition = "2018" +publish = false + +[dependencies] +trace-var = { path = "../trace-var" } diff --git a/syn/examples/trace-var/example/src/main.rs b/syn/examples/trace-var/example/src/main.rs new file mode 100644 index 0000000..da2c10b --- /dev/null +++ b/syn/examples/trace-var/example/src/main.rs @@ -0,0 +1,15 @@ +use trace_var::trace_var; + +fn main() { + println!("{}", factorial(8)); +} + +#[trace_var(p, n)] +fn factorial(mut n: u64) -> u64 { + let mut p = 1; + while n > 1 { + p *= n; + n -= 1; + } + p +} diff --git a/syn/examples/trace-var/trace-var/Cargo.toml b/syn/examples/trace-var/trace-var/Cargo.toml new file mode 100644 index 0000000..72f56e9 --- /dev/null +++ b/syn/examples/trace-var/trace-var/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "trace-var" +version = "0.0.0" +authors = ["David Tolnay "] +edition = "2018" +publish = false + +[lib] +proc-macro = true + +[dependencies] +proc-macro2 = { version = "1.0", features = ["nightly"] } +quote = "1.0" +syn = { path = "../../../", features = ["full", "fold"] } diff --git a/syn/examples/trace-var/trace-var/src/lib.rs b/syn/examples/trace-var/trace-var/src/lib.rs new file mode 100644 index 0000000..0ecfb47 --- /dev/null +++ b/syn/examples/trace-var/trace-var/src/lib.rs @@ -0,0 +1,180 @@ +extern crate proc_macro; +use self::proc_macro::TokenStream; + +use quote::{quote, ToTokens}; +use std::collections::HashSet as Set; +use syn::fold::{self, Fold}; +use syn::parse::{Parse, ParseStream, Result}; +use syn::punctuated::Punctuated; +use syn::{parse_macro_input, parse_quote, Expr, Ident, ItemFn, Local, Pat, Stmt, Token}; + +/// Parses a list of variable names separated by commas. +/// +/// a, b, c +/// +/// This is how the compiler passes in arguments to our attribute -- it is +/// everything inside the delimiters after the attribute name. +/// +/// #[trace_var(a, b, c)] +/// ^^^^^^^ +struct Args { + vars: Set, +} + +impl Parse for Args { + fn parse(input: ParseStream) -> Result { + let vars = Punctuated::::parse_terminated(input)?; + Ok(Args { + vars: vars.into_iter().collect(), + }) + } +} + +impl Args { + /// Determines whether the given `Expr` is a path referring to one of the + /// variables we intend to print. Expressions are used as the left-hand side + /// of the assignment operator. + fn should_print_expr(&self, e: &Expr) -> bool { + match *e { + Expr::Path(ref e) => { + if e.path.leading_colon.is_some() { + false + } else if e.path.segments.len() != 1 { + false + } else { + let first = e.path.segments.first().unwrap(); + self.vars.contains(&first.ident) && first.arguments.is_empty() + } + } + _ => false, + } + } + + /// Determines whether the given `Pat` is an identifier equal to one of the + /// variables we intend to print. Patterns are used as the left-hand side of + /// a `let` binding. + fn should_print_pat(&self, p: &Pat) -> bool { + match p { + Pat::Ident(ref p) => self.vars.contains(&p.ident), + _ => false, + } + } + + /// Produces an expression that assigns the right-hand side to the left-hand + /// side and then prints the value. + /// + /// // Before + /// VAR = INIT + /// + /// // After + /// { VAR = INIT; println!("VAR = {:?}", VAR); } + fn assign_and_print(&mut self, left: Expr, op: &dyn ToTokens, right: Expr) -> Expr { + let right = fold::fold_expr(self, right); + parse_quote!({ + #left #op #right; + println!(concat!(stringify!(#left), " = {:?}"), #left); + }) + } + + /// Produces a let-binding that assigns the right-hand side to the left-hand + /// side and then prints the value. + /// + /// // Before + /// let VAR = INIT; + /// + /// // After + /// let VAR = { let VAR = INIT; println!("VAR = {:?}", VAR); VAR }; + fn let_and_print(&mut self, local: Local) -> Stmt { + let Local { pat, init, .. } = local; + let init = self.fold_expr(*init.unwrap().1); + let ident = match pat { + Pat::Ident(ref p) => &p.ident, + _ => unreachable!(), + }; + parse_quote! { + let #pat = { + #[allow(unused_mut)] + let #pat = #init; + println!(concat!(stringify!(#ident), " = {:?}"), #ident); + #ident + }; + } + } +} + +/// The `Fold` trait is a way to traverse an owned syntax tree and replace some +/// of its nodes. +/// +/// Syn provides two other syntax tree traversal traits: `Visit` which walks a +/// shared borrow of a syntax tree, and `VisitMut` which walks an exclusive +/// borrow of a syntax tree and can mutate it in place. +/// +/// All three traits have a method corresponding to each type of node in Syn's +/// syntax tree. All of these methods have default no-op implementations that +/// simply recurse on any child nodes. We can override only those methods for +/// which we want non-default behavior. In this case the traversal needs to +/// transform `Expr` and `Stmt` nodes. +impl Fold for Args { + fn fold_expr(&mut self, e: Expr) -> Expr { + match e { + Expr::Assign(e) => { + if self.should_print_expr(&e.left) { + self.assign_and_print(*e.left, &e.eq_token, *e.right) + } else { + Expr::Assign(fold::fold_expr_assign(self, e)) + } + } + Expr::AssignOp(e) => { + if self.should_print_expr(&e.left) { + self.assign_and_print(*e.left, &e.op, *e.right) + } else { + Expr::AssignOp(fold::fold_expr_assign_op(self, e)) + } + } + _ => fold::fold_expr(self, e), + } + } + + fn fold_stmt(&mut self, s: Stmt) -> Stmt { + match s { + Stmt::Local(s) => { + if s.init.is_some() && self.should_print_pat(&s.pat) { + self.let_and_print(s) + } else { + Stmt::Local(fold::fold_local(self, s)) + } + } + _ => fold::fold_stmt(self, s), + } + } +} + +/// Attribute to print the value of the given variables each time they are +/// reassigned. +/// +/// # Example +/// +/// ``` +/// #[trace_var(p, n)] +/// fn factorial(mut n: u64) -> u64 { +/// let mut p = 1; +/// while n > 1 { +/// p *= n; +/// n -= 1; +/// } +/// p +/// } +/// ``` +#[proc_macro_attribute] +pub fn trace_var(args: TokenStream, input: TokenStream) -> TokenStream { + let input = parse_macro_input!(input as ItemFn); + + // Parse the list of variables the user wanted to print. + let mut args = parse_macro_input!(args as Args); + + // Use a syntax tree traversal to transform the function body. + let output = args.fold_item_fn(input); + + // Hand the resulting function body back to the compiler. + TokenStream::from(quote!(#output)) +} diff --git a/syn/json/Cargo.toml b/syn/json/Cargo.toml new file mode 100644 index 0000000..77104dc --- /dev/null +++ b/syn/json/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "syn-codegen" +version = "0.1.0" +authors = ["David Tolnay "] +edition = "2018" +license = "MIT OR Apache-2.0" +description = "Syntax tree describing Syn's syntax tree" +repository = "https://github.com/dtolnay/syn" +documentation = "https://docs.rs/syn-codegen" +categories = ["development-tools::procedural-macro-helpers"] + +[dependencies] +indexmap = { version = "1.0", features = ["serde-1"] } +semver = { version = "0.9", features = ["serde"] } +serde = { version = "1.0.88", features = ["derive"] } + +[dev-dependencies] +serde_json = "1.0" diff --git a/syn/json/src/lib.rs b/syn/json/src/lib.rs new file mode 100644 index 0000000..e3546d9 --- /dev/null +++ b/syn/json/src/lib.rs @@ -0,0 +1,214 @@ +//! # Data structures that describe Syn's syntax tree. +//! +//! The Syn syntax tree is made up of more than 200 types. Occasionally it can +//! come up that you need to implement some behavior across them all. +//! +//! - For example [the Rust integration for AST Explorer][astexplorer] wants to +//! turn a syntax tree from Syn into a JavaScript value understood by the +//! platform's existing cross-language syntax tree visualization code. +//! +//! [astexplorer]: https://astexplorer.net/#/gist/388150a52f74d45a355d2b5e865ded96/0c6d563f28d900472f699c21a1845ad20ae9927f +//! +//! - As another example from within Syn itself, the traits and implementations +//! of the [`visit`], [`visit_mut`], and [`fold`] modules can be generated +//! programmatically from a description of the syntax tree. +//! +//! [`visit`]: https://docs.rs/syn/1.0/syn/visit/index.html +//! [`visit_mut`]: https://docs.rs/syn/1.0/syn/visit_mut/index.html +//! [`fold`]: https://docs.rs/syn/1.0/syn/fold/index.html +//! +//! To make this type of code as easy as possible to implement in any language, +//! every Syn release comes with a machine-readable description of that version +//! of the syntax tree as a JSON file [syn.json]. This `syn-codegen` crate +//! provides the canonical data structures for parsing and making use of the +//! representation in syn.json from Rust code. +//! +//! [syn.json]: https://raw.githubusercontent.com/dtolnay/syn/master/syn.json +//! +//! ## Example +//! +//! ``` +//! use syn_codegen::Definitions; +//! +//! # const IGNORE: &str = stringify! { +//! const SYN: &str = include_str!("syn.json"); +//! # }; +//! # const SYN: &str = include_str!("../../syn.json"); +//! +//! fn main() { +//! let defs: Definitions = serde_json::from_str(SYN).unwrap(); +//! +//! for node in &defs.types { +//! println!("syn::{}", node.ident); +//! } +//! } +//! ``` + +use indexmap::IndexMap; +use semver::Version; +use serde::{Deserialize, Deserializer, Serialize}; + +use std::collections::{BTreeMap, BTreeSet}; + +/// Top-level content of the syntax tree description. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct Definitions { + /// The Syn version whose syntax tree is described by this data. + pub version: Version, + + /// Syntax tree types defined by Syn. + pub types: Vec, + + /// Token types defined by Syn (keywords as well as punctuation). + /// + /// The keys in the map are the Rust type name for the token. The values in + /// the map are the printed token representation. + /// + /// These tokens are accessible in the Syn public API as `syn::token::#name` + /// or alternatively `syn::Token![#repr]`. + pub tokens: BTreeMap, +} + +/// Syntax tree type defined by Syn. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct Node { + /// Name of the type. + /// + /// This type is accessible in the Syn public API as `syn::#name`. + pub ident: String, + + /// Features behind which this type is cfg gated. + pub features: Features, + + /// Content of the data structure. + #[serde( + flatten, + skip_serializing_if = "is_private", + deserialize_with = "private_if_absent" + )] + pub data: Data, + + #[serde(skip_serializing_if = "is_true", default = "bool_true")] + pub exhaustive: bool, +} + +/// Content of a syntax tree data structure. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub enum Data { + /// This is an opaque type with no publicy accessible structure. + Private, + + /// This type is a braced struct with named fields. + #[serde(rename = "fields")] + Struct(Fields), + + /// This type is an enum. + #[serde(rename = "variants")] + Enum(Variants), +} + +/// Fields of a braced struct syntax tree node with named fields. +/// +/// The keys in the map are the field names. +pub type Fields = IndexMap; + +/// Variants of an enum syntax tree node. +/// +/// The keys in the map are the variant names. +/// +/// Variants are unit variants if they hold no data and tuple variants +/// otherwise. The Syn syntax tree does not make use of braced variants. +pub type Variants = IndexMap>; + +/// Type of a struct field or tuple variant field in the syntax tree. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum Type { + /// Syntax tree type defined by Syn. + /// + /// This name will match the ident of some `Node`. + Syn(String), + + /// Type defined by the Rust language or standard library. + /// + /// All such types used by Syn are accessible in the Rust prelude and can be + /// used without a qualifying path in most Rust code. + Std(String), + + /// Type defined by proc-macro2. + /// + /// The type is accessible in the proc-macro2 public API as + /// `proc_macro2::#name`. + #[serde(rename = "proc_macro2")] + Ext(String), + + /// Keyword or punctuation token type defined by Syn. + /// + /// This name will match one of the keys in the `tokens` map. + Token(String), + + /// Grouping token defined by Syn. + /// + /// The type is accessible in the Syn public API as `syn::token::#name`. + Group(String), + + /// Punctuated list. + /// + /// This refers to `syn::punctuated::Punctuated` with the specified + /// element type and punctuation. + Punctuated(Punctuated), + + /// `std::option::Option` + Option(Box), + + /// `std::boxed::Box` + Box(Box), + + /// `std::vec::Vec` + Vec(Box), + + /// Rust tuple with two or more fields. + Tuple(Vec), +} + +/// Type of a punctuated list. +/// +/// This refers to `syn::punctuated::Punctuated<#element, #punct>`. +/// +/// The punct string will match one of the keys in the `tokens` map. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct Punctuated { + pub element: Box, + pub punct: String, +} + +/// Features behind which a syntax tree type is cfg gated. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +pub struct Features { + /// Type is accessible if at least one of these features is enabled against + /// the Syn dependency. + pub any: BTreeSet, +} + +fn is_private(data: &Data) -> bool { + match data { + Data::Private => true, + Data::Struct(_) | Data::Enum(_) => false, + } +} + +fn private_if_absent<'de, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, +{ + let option = Option::deserialize(deserializer)?; + Ok(option.unwrap_or(Data::Private)) +} + +fn is_true(b: &bool) -> bool { + *b +} + +fn bool_true() -> bool { + true +} diff --git a/syn/src/attr.rs b/syn/src/attr.rs new file mode 100644 index 0000000..a8e16ea --- /dev/null +++ b/syn/src/attr.rs @@ -0,0 +1,682 @@ +use super::*; +use crate::punctuated::Punctuated; + +use std::iter; + +use proc_macro2::TokenStream; + +#[cfg(feature = "parsing")] +use crate::parse::{Parse, ParseBuffer, ParseStream, Parser, Result}; +#[cfg(feature = "parsing")] +use crate::punctuated::Pair; +#[cfg(feature = "extra-traits")] +use crate::tt::TokenStreamHelper; +#[cfg(feature = "extra-traits")] +use std::hash::{Hash, Hasher}; + +ast_struct! { + /// An attribute like `#[repr(transparent)]`. + /// + /// *This type is available if Syn is built with the `"derive"` or `"full"` + /// feature.* + /// + ///
+ /// + /// # Syntax + /// + /// Rust has six types of attributes. + /// + /// - Outer attributes like `#[repr(transparent)]`. These appear outside or + /// in front of the item they describe. + /// - Inner attributes like `#![feature(proc_macro)]`. These appear inside + /// of the item they describe, usually a module. + /// - Outer doc comments like `/// # Example`. + /// - Inner doc comments like `//! Please file an issue`. + /// - Outer block comments `/** # Example */`. + /// - Inner block comments `/*! Please file an issue */`. + /// + /// The `style` field of type `AttrStyle` distinguishes whether an attribute + /// is outer or inner. Doc comments and block comments are promoted to + /// attributes, as this is how they are processed by the compiler and by + /// `macro_rules!` macros. + /// + /// The `path` field gives the possibly colon-delimited path against which + /// the attribute is resolved. It is equal to `"doc"` for desugared doc + /// comments. The `tokens` field contains the rest of the attribute body as + /// tokens. + /// + /// ```text + /// #[derive(Copy)] #[crate::precondition x < 5] + /// ^^^^^^~~~~~~ ^^^^^^^^^^^^^^^^^^^ ~~~~~ + /// path tokens path tokens + /// ``` + /// + ///
+ /// + /// # Parsing from tokens to Attribute + /// + /// This type does not implement the [`Parse`] trait and thus cannot be + /// parsed directly by [`ParseStream::parse`]. Instead use + /// [`ParseStream::call`] with one of the two parser functions + /// [`Attribute::parse_outer`] or [`Attribute::parse_inner`] depending on + /// which you intend to parse. + /// + /// [`Parse`]: parse::Parse + /// [`ParseStream::parse`]: parse::ParseBuffer::parse + /// [`ParseStream::call`]: parse::ParseBuffer::call + /// + /// ``` + /// use syn::{Attribute, Ident, Result, Token}; + /// use syn::parse::{Parse, ParseStream}; + /// + /// // Parses a unit struct with attributes. + /// // + /// // #[path = "s.tmpl"] + /// // struct S; + /// struct UnitStruct { + /// attrs: Vec, + /// struct_token: Token![struct], + /// name: Ident, + /// semi_token: Token![;], + /// } + /// + /// impl Parse for UnitStruct { + /// fn parse(input: ParseStream) -> Result { + /// Ok(UnitStruct { + /// attrs: input.call(Attribute::parse_outer)?, + /// struct_token: input.parse()?, + /// name: input.parse()?, + /// semi_token: input.parse()?, + /// }) + /// } + /// } + /// ``` + /// + ///


+ /// + /// # Parsing from Attribute to structured arguments + /// + /// The grammar of attributes in Rust is very flexible, which makes the + /// syntax tree not that useful on its own. In particular, arguments of the + /// attribute are held in an arbitrary `tokens: TokenStream`. Macros are + /// expected to check the `path` of the attribute, decide whether they + /// recognize it, and then parse the remaining tokens according to whatever + /// grammar they wish to require for that kind of attribute. + /// + /// If the attribute you are parsing is expected to conform to the + /// conventional structured form of attribute, use [`parse_meta()`] to + /// obtain that structured representation. If the attribute follows some + /// other grammar of its own, use [`parse_args()`] to parse that into the + /// expected data structure. + /// + /// [`parse_meta()`]: Attribute::parse_meta + /// [`parse_args()`]: Attribute::parse_args + /// + ///


+ /// + /// # Doc comments + /// + /// The compiler transforms doc comments, such as `/// comment` and `/*! + /// comment */`, into attributes before macros are expanded. Each comment is + /// expanded into an attribute of the form `#[doc = r"comment"]`. + /// + /// As an example, the following `mod` items are expanded identically: + /// + /// ``` + /// # use syn::{ItemMod, parse_quote}; + /// let doc: ItemMod = parse_quote! { + /// /// Single line doc comments + /// /// We write so many! + /// /** + /// * Multi-line comments... + /// * May span many lines + /// */ + /// mod example { + /// //! Of course, they can be inner too + /// /*! And fit in a single line */ + /// } + /// }; + /// let attr: ItemMod = parse_quote! { + /// #[doc = r" Single line doc comments"] + /// #[doc = r" We write so many!"] + /// #[doc = r" Multi-line comments... + /// May span many lines"] + /// mod example { + /// #![doc = r" Of course, they can be inner too"] + /// #![doc = r" And fit in a single line "] + /// } + /// }; + /// assert_eq!(doc, attr); + /// ``` + pub struct Attribute #manual_extra_traits { + pub pound_token: Token![#], + pub style: AttrStyle, + pub bracket_token: token::Bracket, + pub path: Path, + pub tokens: TokenStream, + } +} + +#[cfg(feature = "extra-traits")] +impl Eq for Attribute {} + +#[cfg(feature = "extra-traits")] +impl PartialEq for Attribute { + fn eq(&self, other: &Self) -> bool { + self.style == other.style + && self.pound_token == other.pound_token + && self.bracket_token == other.bracket_token + && self.path == other.path + && TokenStreamHelper(&self.tokens) == TokenStreamHelper(&other.tokens) + } +} + +#[cfg(feature = "extra-traits")] +impl Hash for Attribute { + fn hash(&self, state: &mut H) + where + H: Hasher, + { + self.style.hash(state); + self.pound_token.hash(state); + self.bracket_token.hash(state); + self.path.hash(state); + TokenStreamHelper(&self.tokens).hash(state); + } +} + +impl Attribute { + /// Parses the content of the attribute, consisting of the path and tokens, + /// as a [`Meta`] if possible. + /// + /// *This function is available if Syn is built with the `"parsing"` + /// feature.* + #[cfg(feature = "parsing")] + pub fn parse_meta(&self) -> Result { + fn clone_ident_segment(segment: &PathSegment) -> PathSegment { + PathSegment { + ident: segment.ident.clone(), + arguments: PathArguments::None, + } + } + + let path = Path { + leading_colon: self + .path + .leading_colon + .as_ref() + .map(|colon| Token![::](colon.spans)), + segments: self + .path + .segments + .pairs() + .map(|pair| match pair { + Pair::Punctuated(seg, punct) => { + Pair::Punctuated(clone_ident_segment(seg), Token![::](punct.spans)) + } + Pair::End(seg) => Pair::End(clone_ident_segment(seg)), + }) + .collect(), + }; + + let parser = |input: ParseStream| parsing::parse_meta_after_path(path, input); + parse::Parser::parse2(parser, self.tokens.clone()) + } + + /// Parse the arguments to the attribute as a syntax tree. + /// + /// This is similar to `syn::parse2::(attr.tokens)` except that: + /// + /// - the surrounding delimiters are *not* included in the input to the + /// parser; and + /// - the error message has a more useful span when `tokens` is empty. + /// + /// ```text + /// #[my_attr(value < 5)] + /// ^^^^^^^^^ what gets parsed + /// ``` + /// + /// *This function is available if Syn is built with the `"parsing"` + /// feature.* + #[cfg(feature = "parsing")] + pub fn parse_args(&self) -> Result { + self.parse_args_with(T::parse) + } + + /// Parse the arguments to the attribute using the given parser. + /// + /// *This function is available if Syn is built with the `"parsing"` + /// feature.* + #[cfg(feature = "parsing")] + pub fn parse_args_with(&self, parser: F) -> Result { + let parser = |input: ParseStream| { + let args = enter_args(self, input)?; + parse::parse_stream(parser, &args) + }; + parser.parse2(self.tokens.clone()) + } + + /// Parses zero or more outer attributes from the stream. + /// + /// *This function is available if Syn is built with the `"parsing"` + /// feature.* + #[cfg(feature = "parsing")] + pub fn parse_outer(input: ParseStream) -> Result> { + let mut attrs = Vec::new(); + while input.peek(Token![#]) { + attrs.push(input.call(parsing::single_parse_outer)?); + } + Ok(attrs) + } + + /// Parses zero or more inner attributes from the stream. + /// + /// *This function is available if Syn is built with the `"parsing"` + /// feature.* + #[cfg(feature = "parsing")] + pub fn parse_inner(input: ParseStream) -> Result> { + let mut attrs = Vec::new(); + while input.peek(Token![#]) && input.peek2(Token![!]) { + attrs.push(input.call(parsing::single_parse_inner)?); + } + Ok(attrs) + } +} + +#[cfg(feature = "parsing")] +fn error_expected_args(attr: &Attribute) -> Error { + let style = match attr.style { + AttrStyle::Outer => "#", + AttrStyle::Inner(_) => "#!", + }; + + let mut path = String::new(); + for segment in &attr.path.segments { + if !path.is_empty() || attr.path.leading_colon.is_some() { + path += "::"; + } + path += &segment.ident.to_string(); + } + + let msg = format!("expected attribute arguments: {}[{}(...)]", style, path); + + #[cfg(feature = "printing")] + return Error::new_spanned(attr, msg); + + #[cfg(not(feature = "printing"))] + return Error::new(attr.bracket_token.span, msg); +} + +#[cfg(feature = "parsing")] +fn enter_args<'a>(attr: &Attribute, input: ParseStream<'a>) -> Result> { + if input.is_empty() { + return Err(error_expected_args(attr)); + }; + + let content; + if input.peek(token::Paren) { + parenthesized!(content in input); + } else if input.peek(token::Bracket) { + bracketed!(content in input); + } else if input.peek(token::Brace) { + braced!(content in input); + } else { + return Err(input.error("unexpected token in attribute arguments")); + } + + if input.is_empty() { + Ok(content) + } else { + Err(input.error("unexpected token in attribute arguments")) + } +} + +ast_enum! { + /// Distinguishes between attributes that decorate an item and attributes + /// that are contained within an item. + /// + /// *This type is available if Syn is built with the `"derive"` or `"full"` + /// feature.* + /// + /// # Outer attributes + /// + /// - `#[repr(transparent)]` + /// - `/// # Example` + /// - `/** Please file an issue */` + /// + /// # Inner attributes + /// + /// - `#![feature(proc_macro)]` + /// - `//! # Example` + /// - `/*! Please file an issue */` + #[cfg_attr(feature = "clone-impls", derive(Copy))] + pub enum AttrStyle { + Outer, + Inner(Token![!]), + } +} + +ast_enum_of_structs! { + /// Content of a compile-time structured attribute. + /// + /// *This type is available if Syn is built with the `"derive"` or `"full"` + /// feature.* + /// + /// ## Path + /// + /// A meta path is like the `test` in `#[test]`. + /// + /// ## List + /// + /// A meta list is like the `derive(Copy)` in `#[derive(Copy)]`. + /// + /// ## NameValue + /// + /// A name-value meta is like the `path = "..."` in `#[path = + /// "sys/windows.rs"]`. + /// + /// # Syntax tree enum + /// + /// This type is a [syntax tree enum]. + /// + /// [syntax tree enum]: enum.Expr.html#syntax-tree-enums + // + // TODO: change syntax-tree-enum link to an intra rustdoc link, currently + // blocked on https://github.com/rust-lang/rust/issues/62833 + pub enum Meta { + Path(Path), + + /// A structured list within an attribute, like `derive(Copy, Clone)`. + List(MetaList), + + /// A name-value pair within an attribute, like `feature = "nightly"`. + NameValue(MetaNameValue), + } +} + +ast_struct! { + /// A structured list within an attribute, like `derive(Copy, Clone)`. + /// + /// *This type is available if Syn is built with the `"derive"` or + /// `"full"` feature.* + pub struct MetaList { + pub path: Path, + pub paren_token: token::Paren, + pub nested: Punctuated, + } +} + +ast_struct! { + /// A name-value pair within an attribute, like `feature = "nightly"`. + /// + /// *This type is available if Syn is built with the `"derive"` or + /// `"full"` feature.* + pub struct MetaNameValue { + pub path: Path, + pub eq_token: Token![=], + pub lit: Lit, + } +} + +impl Meta { + /// Returns the identifier that begins this structured meta item. + /// + /// For example this would return the `test` in `#[test]`, the `derive` in + /// `#[derive(Copy)]`, and the `path` in `#[path = "sys/windows.rs"]`. + pub fn path(&self) -> &Path { + match self { + Meta::Path(path) => path, + Meta::List(meta) => &meta.path, + Meta::NameValue(meta) => &meta.path, + } + } +} + +ast_enum_of_structs! { + /// Element of a compile-time attribute list. + /// + /// *This type is available if Syn is built with the `"derive"` or `"full"` + /// feature.* + pub enum NestedMeta { + /// A structured meta item, like the `Copy` in `#[derive(Copy)]` which + /// would be a nested `Meta::Path`. + Meta(Meta), + + /// A Rust literal, like the `"new_name"` in `#[rename("new_name")]`. + Lit(Lit), + } +} + +/// Conventional argument type associated with an invocation of an attribute +/// macro. +/// +/// For example if we are developing an attribute macro that is intended to be +/// invoked on function items as follows: +/// +/// ``` +/// # const IGNORE: &str = stringify! { +/// #[my_attribute(path = "/v1/refresh")] +/// # }; +/// pub fn refresh() { +/// /* ... */ +/// } +/// ``` +/// +/// The implementation of this macro would want to parse its attribute arguments +/// as type `AttributeArgs`. +/// +/// ``` +/// extern crate proc_macro; +/// +/// use proc_macro::TokenStream; +/// use syn::{parse_macro_input, AttributeArgs, ItemFn}; +/// +/// # const IGNORE: &str = stringify! { +/// #[proc_macro_attribute] +/// # }; +/// pub fn my_attribute(args: TokenStream, input: TokenStream) -> TokenStream { +/// let args = parse_macro_input!(args as AttributeArgs); +/// let input = parse_macro_input!(input as ItemFn); +/// +/// /* ... */ +/// # "".parse().unwrap() +/// } +/// ``` +pub type AttributeArgs = Vec; + +pub trait FilterAttrs<'a> { + type Ret: Iterator; + + fn outer(self) -> Self::Ret; + fn inner(self) -> Self::Ret; +} + +impl<'a, T> FilterAttrs<'a> for T +where + T: IntoIterator, +{ + type Ret = iter::Filter bool>; + + fn outer(self) -> Self::Ret { + fn is_outer(attr: &&Attribute) -> bool { + match attr.style { + AttrStyle::Outer => true, + _ => false, + } + } + self.into_iter().filter(is_outer) + } + + fn inner(self) -> Self::Ret { + fn is_inner(attr: &&Attribute) -> bool { + match attr.style { + AttrStyle::Inner(_) => true, + _ => false, + } + } + self.into_iter().filter(is_inner) + } +} + +#[cfg(feature = "parsing")] +pub mod parsing { + use super::*; + + use crate::ext::IdentExt; + use crate::parse::{Parse, ParseStream, Result}; + #[cfg(feature = "full")] + use crate::private; + + pub fn single_parse_inner(input: ParseStream) -> Result { + let content; + Ok(Attribute { + pound_token: input.parse()?, + style: AttrStyle::Inner(input.parse()?), + bracket_token: bracketed!(content in input), + path: content.call(Path::parse_mod_style)?, + tokens: content.parse()?, + }) + } + + pub fn single_parse_outer(input: ParseStream) -> Result { + let content; + Ok(Attribute { + pound_token: input.parse()?, + style: AttrStyle::Outer, + bracket_token: bracketed!(content in input), + path: content.call(Path::parse_mod_style)?, + tokens: content.parse()?, + }) + } + + #[cfg(feature = "full")] + impl private { + pub fn attrs(outer: Vec, inner: Vec) -> Vec { + let mut attrs = outer; + attrs.extend(inner); + attrs + } + } + + // Like Path::parse_mod_style but accepts keywords in the path. + fn parse_meta_path(input: ParseStream) -> Result { + Ok(Path { + leading_colon: input.parse()?, + segments: { + let mut segments = Punctuated::new(); + while input.peek(Ident::peek_any) { + let ident = Ident::parse_any(input)?; + segments.push_value(PathSegment::from(ident)); + if !input.peek(Token![::]) { + break; + } + let punct = input.parse()?; + segments.push_punct(punct); + } + if segments.is_empty() { + return Err(input.error("expected path")); + } else if segments.trailing_punct() { + return Err(input.error("expected path segment")); + } + segments + }, + }) + } + + impl Parse for Meta { + fn parse(input: ParseStream) -> Result { + let path = input.call(parse_meta_path)?; + parse_meta_after_path(path, input) + } + } + + impl Parse for MetaList { + fn parse(input: ParseStream) -> Result { + let path = input.call(parse_meta_path)?; + parse_meta_list_after_path(path, input) + } + } + + impl Parse for MetaNameValue { + fn parse(input: ParseStream) -> Result { + let path = input.call(parse_meta_path)?; + parse_meta_name_value_after_path(path, input) + } + } + + impl Parse for NestedMeta { + fn parse(input: ParseStream) -> Result { + if input.peek(Lit) && !(input.peek(LitBool) && input.peek2(Token![=])) { + input.parse().map(NestedMeta::Lit) + } else if input.peek(Ident::peek_any) { + input.parse().map(NestedMeta::Meta) + } else { + Err(input.error("expected identifier or literal")) + } + } + } + + pub fn parse_meta_after_path(path: Path, input: ParseStream) -> Result { + if input.peek(token::Paren) { + parse_meta_list_after_path(path, input).map(Meta::List) + } else if input.peek(Token![=]) { + parse_meta_name_value_after_path(path, input).map(Meta::NameValue) + } else { + Ok(Meta::Path(path)) + } + } + + fn parse_meta_list_after_path(path: Path, input: ParseStream) -> Result { + let content; + Ok(MetaList { + path, + paren_token: parenthesized!(content in input), + nested: content.parse_terminated(NestedMeta::parse)?, + }) + } + + fn parse_meta_name_value_after_path(path: Path, input: ParseStream) -> Result { + Ok(MetaNameValue { + path, + eq_token: input.parse()?, + lit: input.parse()?, + }) + } +} + +#[cfg(feature = "printing")] +mod printing { + use super::*; + use proc_macro2::TokenStream; + use quote::ToTokens; + + impl ToTokens for Attribute { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.pound_token.to_tokens(tokens); + if let AttrStyle::Inner(b) = &self.style { + b.to_tokens(tokens); + } + self.bracket_token.surround(tokens, |tokens| { + self.path.to_tokens(tokens); + self.tokens.to_tokens(tokens); + }); + } + } + + impl ToTokens for MetaList { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.path.to_tokens(tokens); + self.paren_token.surround(tokens, |tokens| { + self.nested.to_tokens(tokens); + }) + } + } + + impl ToTokens for MetaNameValue { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.path.to_tokens(tokens); + self.eq_token.to_tokens(tokens); + self.lit.to_tokens(tokens); + } + } +} diff --git a/syn/src/await.rs b/syn/src/await.rs new file mode 100644 index 0000000..a8e24fd --- /dev/null +++ b/syn/src/await.rs @@ -0,0 +1,2 @@ +// See include!("await.rs") in token.rs. +export_token_macro![(await)]; diff --git a/syn/src/bigint.rs b/syn/src/bigint.rs new file mode 100644 index 0000000..5397d6b --- /dev/null +++ b/syn/src/bigint.rs @@ -0,0 +1,66 @@ +use std::ops::{AddAssign, MulAssign}; + +// For implementing base10_digits() accessor on LitInt. +pub struct BigInt { + digits: Vec, +} + +impl BigInt { + pub fn new() -> Self { + BigInt { digits: Vec::new() } + } + + pub fn to_string(&self) -> String { + let mut repr = String::with_capacity(self.digits.len()); + + let mut has_nonzero = false; + for digit in self.digits.iter().rev() { + has_nonzero |= *digit != 0; + if has_nonzero { + repr.push((*digit + b'0') as char); + } + } + + if repr.is_empty() { + repr.push('0'); + } + + repr + } + + fn reserve_two_digits(&mut self) { + let len = self.digits.len(); + let desired = + len + !self.digits.ends_with(&[0, 0]) as usize + !self.digits.ends_with(&[0]) as usize; + self.digits.resize(desired, 0); + } +} + +impl AddAssign for BigInt { + // Assumes increment <16. + fn add_assign(&mut self, mut increment: u8) { + self.reserve_two_digits(); + + let mut i = 0; + while increment > 0 { + let sum = self.digits[i] + increment; + self.digits[i] = sum % 10; + increment = sum / 10; + i += 1; + } + } +} + +impl MulAssign for BigInt { + // Assumes base <=16. + fn mul_assign(&mut self, base: u8) { + self.reserve_two_digits(); + + let mut carry = 0; + for digit in &mut self.digits { + let prod = *digit * base + carry; + *digit = prod % 10; + carry = prod / 10; + } + } +} diff --git a/syn/src/buffer.rs b/syn/src/buffer.rs new file mode 100644 index 0000000..5c2dd8a --- /dev/null +++ b/syn/src/buffer.rs @@ -0,0 +1,382 @@ +//! A stably addressed token buffer supporting efficient traversal based on a +//! cheaply copyable cursor. +//! +//! *This module is available if Syn is built with the `"parsing"` feature.* + +// This module is heavily commented as it contains most of the unsafe code in +// Syn, and caution should be used when editing it. The public-facing interface +// is 100% safe but the implementation is fragile internally. + +#[cfg(all( + not(all(target_arch = "wasm32", any(target_os = "unknown", target_os = "wasi"))), + feature = "proc-macro" +))] +use crate::proc_macro as pm; +use proc_macro2::{Delimiter, Group, Ident, Literal, Punct, Spacing, Span, TokenStream, TokenTree}; + +use std::marker::PhantomData; +use std::ptr; + +use crate::Lifetime; + +/// Internal type which is used instead of `TokenTree` to represent a token tree +/// within a `TokenBuffer`. +enum Entry { + // Mimicking types from proc-macro. + Group(Group, TokenBuffer), + Ident(Ident), + Punct(Punct), + Literal(Literal), + // End entries contain a raw pointer to the entry from the containing + // token tree, or null if this is the outermost level. + End(*const Entry), +} + +/// A buffer that can be efficiently traversed multiple times, unlike +/// `TokenStream` which requires a deep copy in order to traverse more than +/// once. +/// +/// *This type is available if Syn is built with the `"parsing"` feature.* +pub struct TokenBuffer { + // NOTE: Do not derive clone on this - there are raw pointers inside which + // will be messed up. Moving the `TokenBuffer` itself is safe as the actual + // backing slices won't be moved. + data: Box<[Entry]>, +} + +impl TokenBuffer { + // NOTE: DO NOT MUTATE THE `Vec` RETURNED FROM THIS FUNCTION ONCE IT + // RETURNS, THE ADDRESS OF ITS BACKING MEMORY MUST REMAIN STABLE. + fn inner_new(stream: TokenStream, up: *const Entry) -> TokenBuffer { + // Build up the entries list, recording the locations of any Groups + // in the list to be processed later. + let mut entries = Vec::new(); + let mut seqs = Vec::new(); + for tt in stream { + match tt { + TokenTree::Ident(sym) => { + entries.push(Entry::Ident(sym)); + } + TokenTree::Punct(op) => { + entries.push(Entry::Punct(op)); + } + TokenTree::Literal(l) => { + entries.push(Entry::Literal(l)); + } + TokenTree::Group(g) => { + // Record the index of the interesting entry, and store an + // `End(null)` there temporarially. + seqs.push((entries.len(), g)); + entries.push(Entry::End(ptr::null())); + } + } + } + // Add an `End` entry to the end with a reference to the enclosing token + // stream which was passed in. + entries.push(Entry::End(up)); + + // NOTE: This is done to ensure that we don't accidentally modify the + // length of the backing buffer. The backing buffer must remain at a + // constant address after this point, as we are going to store a raw + // pointer into it. + let mut entries = entries.into_boxed_slice(); + for (idx, group) in seqs { + // We know that this index refers to one of the temporary + // `End(null)` entries, and we know that the last entry is + // `End(up)`, so the next index is also valid. + let seq_up = &entries[idx + 1] as *const Entry; + + // The end entry stored at the end of this Entry::Group should + // point to the Entry which follows the Group in the list. + let inner = Self::inner_new(group.stream(), seq_up); + entries[idx] = Entry::Group(group, inner); + } + + TokenBuffer { data: entries } + } + + /// Creates a `TokenBuffer` containing all the tokens from the input + /// `TokenStream`. + /// + /// *This method is available if Syn is built with both the `"parsing"` and + /// `"proc-macro"` features.* + #[cfg(all( + not(all(target_arch = "wasm32", any(target_os = "unknown", target_os = "wasi"))), + feature = "proc-macro" + ))] + pub fn new(stream: pm::TokenStream) -> TokenBuffer { + Self::new2(stream.into()) + } + + /// Creates a `TokenBuffer` containing all the tokens from the input + /// `TokenStream`. + pub fn new2(stream: TokenStream) -> TokenBuffer { + Self::inner_new(stream, ptr::null()) + } + + /// Creates a cursor referencing the first token in the buffer and able to + /// traverse until the end of the buffer. + pub fn begin(&self) -> Cursor { + unsafe { Cursor::create(&self.data[0], &self.data[self.data.len() - 1]) } + } +} + +/// A cheaply copyable cursor into a `TokenBuffer`. +/// +/// This cursor holds a shared reference into the immutable data which is used +/// internally to represent a `TokenStream`, and can be efficiently manipulated +/// and copied around. +/// +/// An empty `Cursor` can be created directly, or one may create a `TokenBuffer` +/// object and get a cursor to its first token with `begin()`. +/// +/// Two cursors are equal if they have the same location in the same input +/// stream, and have the same scope. +/// +/// *This type is available if Syn is built with the `"parsing"` feature.* +#[derive(Copy, Clone, Eq, PartialEq)] +pub struct Cursor<'a> { + // The current entry which the `Cursor` is pointing at. + ptr: *const Entry, + // This is the only `Entry::End(..)` object which this cursor is allowed to + // point at. All other `End` objects are skipped over in `Cursor::create`. + scope: *const Entry, + // Cursor is covariant in 'a. This field ensures that our pointers are still + // valid. + marker: PhantomData<&'a Entry>, +} + +impl<'a> Cursor<'a> { + /// Creates a cursor referencing a static empty TokenStream. + pub fn empty() -> Self { + // It's safe in this situation for us to put an `Entry` object in global + // storage, despite it not actually being safe to send across threads + // (`Ident` is a reference into a thread-local table). This is because + // this entry never includes a `Ident` object. + // + // This wrapper struct allows us to break the rules and put a `Sync` + // object in global storage. + struct UnsafeSyncEntry(Entry); + unsafe impl Sync for UnsafeSyncEntry {} + static EMPTY_ENTRY: UnsafeSyncEntry = UnsafeSyncEntry(Entry::End(0 as *const Entry)); + + Cursor { + ptr: &EMPTY_ENTRY.0, + scope: &EMPTY_ENTRY.0, + marker: PhantomData, + } + } + + /// This create method intelligently exits non-explicitly-entered + /// `None`-delimited scopes when the cursor reaches the end of them, + /// allowing for them to be treated transparently. + unsafe fn create(mut ptr: *const Entry, scope: *const Entry) -> Self { + // NOTE: If we're looking at a `End(..)`, we want to advance the cursor + // past it, unless `ptr == scope`, which means that we're at the edge of + // our cursor's scope. We should only have `ptr != scope` at the exit + // from None-delimited groups entered with `ignore_none`. + while let Entry::End(exit) = *ptr { + if ptr == scope { + break; + } + ptr = exit; + } + + Cursor { + ptr, + scope, + marker: PhantomData, + } + } + + /// Get the current entry. + fn entry(self) -> &'a Entry { + unsafe { &*self.ptr } + } + + /// Bump the cursor to point at the next token after the current one. This + /// is undefined behavior if the cursor is currently looking at an + /// `Entry::End`. + unsafe fn bump(self) -> Cursor<'a> { + Cursor::create(self.ptr.offset(1), self.scope) + } + + /// If the cursor is looking at a `None`-delimited group, move it to look at + /// the first token inside instead. If the group is empty, this will move + /// the cursor past the `None`-delimited group. + /// + /// WARNING: This mutates its argument. + fn ignore_none(&mut self) { + if let Entry::Group(group, buf) = self.entry() { + if group.delimiter() == Delimiter::None { + // NOTE: We call `Cursor::create` here to make sure that + // situations where we should immediately exit the span after + // entering it are handled correctly. + unsafe { + *self = Cursor::create(&buf.data[0], self.scope); + } + } + } + } + + /// Checks whether the cursor is currently pointing at the end of its valid + /// scope. + pub fn eof(self) -> bool { + // We're at eof if we're at the end of our scope. + self.ptr == self.scope + } + + /// If the cursor is pointing at a `Group` with the given delimiter, returns + /// a cursor into that group and one pointing to the next `TokenTree`. + pub fn group(mut self, delim: Delimiter) -> Option<(Cursor<'a>, Span, Cursor<'a>)> { + // If we're not trying to enter a none-delimited group, we want to + // ignore them. We have to make sure to _not_ ignore them when we want + // to enter them, of course. For obvious reasons. + if delim != Delimiter::None { + self.ignore_none(); + } + + if let Entry::Group(group, buf) = self.entry() { + if group.delimiter() == delim { + return Some((buf.begin(), group.span(), unsafe { self.bump() })); + } + } + + None + } + + /// If the cursor is pointing at a `Ident`, returns it along with a cursor + /// pointing at the next `TokenTree`. + pub fn ident(mut self) -> Option<(Ident, Cursor<'a>)> { + self.ignore_none(); + match self.entry() { + Entry::Ident(ident) => Some((ident.clone(), unsafe { self.bump() })), + _ => None, + } + } + + /// If the cursor is pointing at an `Punct`, returns it along with a cursor + /// pointing at the next `TokenTree`. + pub fn punct(mut self) -> Option<(Punct, Cursor<'a>)> { + self.ignore_none(); + match self.entry() { + Entry::Punct(op) if op.as_char() != '\'' => Some((op.clone(), unsafe { self.bump() })), + _ => None, + } + } + + /// If the cursor is pointing at a `Literal`, return it along with a cursor + /// pointing at the next `TokenTree`. + pub fn literal(mut self) -> Option<(Literal, Cursor<'a>)> { + self.ignore_none(); + match self.entry() { + Entry::Literal(lit) => Some((lit.clone(), unsafe { self.bump() })), + _ => None, + } + } + + /// If the cursor is pointing at a `Lifetime`, returns it along with a + /// cursor pointing at the next `TokenTree`. + pub fn lifetime(mut self) -> Option<(Lifetime, Cursor<'a>)> { + self.ignore_none(); + match self.entry() { + Entry::Punct(op) if op.as_char() == '\'' && op.spacing() == Spacing::Joint => { + let next = unsafe { self.bump() }; + match next.ident() { + Some((ident, rest)) => { + let lifetime = Lifetime { + apostrophe: op.span(), + ident, + }; + Some((lifetime, rest)) + } + None => None, + } + } + _ => None, + } + } + + /// Copies all remaining tokens visible from this cursor into a + /// `TokenStream`. + pub fn token_stream(self) -> TokenStream { + let mut tts = Vec::new(); + let mut cursor = self; + while let Some((tt, rest)) = cursor.token_tree() { + tts.push(tt); + cursor = rest; + } + tts.into_iter().collect() + } + + /// If the cursor is pointing at a `TokenTree`, returns it along with a + /// cursor pointing at the next `TokenTree`. + /// + /// Returns `None` if the cursor has reached the end of its stream. + /// + /// This method does not treat `None`-delimited groups as transparent, and + /// will return a `Group(None, ..)` if the cursor is looking at one. + pub fn token_tree(self) -> Option<(TokenTree, Cursor<'a>)> { + let tree = match self.entry() { + Entry::Group(group, _) => group.clone().into(), + Entry::Literal(lit) => lit.clone().into(), + Entry::Ident(ident) => ident.clone().into(), + Entry::Punct(op) => op.clone().into(), + Entry::End(..) => { + return None; + } + }; + + Some((tree, unsafe { self.bump() })) + } + + /// Returns the `Span` of the current token, or `Span::call_site()` if this + /// cursor points to eof. + pub fn span(self) -> Span { + match self.entry() { + Entry::Group(group, _) => group.span(), + Entry::Literal(l) => l.span(), + Entry::Ident(t) => t.span(), + Entry::Punct(o) => o.span(), + Entry::End(..) => Span::call_site(), + } + } + + /// Skip over the next token without cloning it. Returns `None` if this + /// cursor points to eof. + /// + /// This method treats `'lifetimes` as a single token. + pub(crate) fn skip(self) -> Option> { + match self.entry() { + Entry::End(..) => None, + + // Treat lifetimes as a single tt for the purposes of 'skip'. + Entry::Punct(op) if op.as_char() == '\'' && op.spacing() == Spacing::Joint => { + let next = unsafe { self.bump() }; + match next.entry() { + Entry::Ident(_) => Some(unsafe { next.bump() }), + _ => Some(next), + } + } + _ => Some(unsafe { self.bump() }), + } + } +} + +pub(crate) fn same_scope(a: Cursor, b: Cursor) -> bool { + a.scope == b.scope +} + +pub(crate) fn open_span_of_group(cursor: Cursor) -> Span { + match cursor.entry() { + Entry::Group(group, _) => group.span_open(), + _ => cursor.span(), + } +} + +pub(crate) fn close_span_of_group(cursor: Cursor) -> Span { + match cursor.entry() { + Entry::Group(group, _) => group.span_close(), + _ => cursor.span(), + } +} diff --git a/syn/src/custom_keyword.rs b/syn/src/custom_keyword.rs new file mode 100644 index 0000000..200e847 --- /dev/null +++ b/syn/src/custom_keyword.rs @@ -0,0 +1,252 @@ +/// Define a type that supports parsing and printing a given identifier as if it +/// were a keyword. +/// +/// # Usage +/// +/// As a convention, it is recommended that this macro be invoked within a +/// module called `kw` or `keyword` and that the resulting parser be invoked +/// with a `kw::` or `keyword::` prefix. +/// +/// ``` +/// mod kw { +/// syn::custom_keyword!(whatever); +/// } +/// ``` +/// +/// The generated syntax tree node supports the following operations just like +/// any built-in keyword token. +/// +/// - [Peeking] — `input.peek(kw::whatever)` +/// +/// - [Parsing] — `input.parse::()?` +/// +/// - [Printing] — `quote!( ... #whatever_token ... )` +/// +/// - Construction from a [`Span`] — `let whatever_token = kw::whatever(sp)` +/// +/// - Field access to its span — `let sp = whatever_token.span` +/// +/// [Peeking]: parse::ParseBuffer::peek +/// [Parsing]: parse::ParseBuffer::parse +/// [Printing]: quote::ToTokens +/// [`Span`]: proc_macro2::Span +/// +/// # Example +/// +/// This example parses input that looks like `bool = true` or `str = "value"`. +/// The key must be either the identifier `bool` or the identifier `str`. If +/// `bool`, the value may be either `true` or `false`. If `str`, the value may +/// be any string literal. +/// +/// The symbols `bool` and `str` are not reserved keywords in Rust so these are +/// not considered keywords in the `syn::token` module. Like any other +/// identifier that is not a keyword, these can be declared as custom keywords +/// by crates that need to use them as such. +/// +/// ``` +/// use syn::{LitBool, LitStr, Result, Token}; +/// use syn::parse::{Parse, ParseStream}; +/// +/// mod kw { +/// syn::custom_keyword!(bool); +/// syn::custom_keyword!(str); +/// } +/// +/// enum Argument { +/// Bool { +/// bool_token: kw::bool, +/// eq_token: Token![=], +/// value: LitBool, +/// }, +/// Str { +/// str_token: kw::str, +/// eq_token: Token![=], +/// value: LitStr, +/// }, +/// } +/// +/// impl Parse for Argument { +/// fn parse(input: ParseStream) -> Result { +/// let lookahead = input.lookahead1(); +/// if lookahead.peek(kw::bool) { +/// Ok(Argument::Bool { +/// bool_token: input.parse::()?, +/// eq_token: input.parse()?, +/// value: input.parse()?, +/// }) +/// } else if lookahead.peek(kw::str) { +/// Ok(Argument::Str { +/// str_token: input.parse::()?, +/// eq_token: input.parse()?, +/// value: input.parse()?, +/// }) +/// } else { +/// Err(lookahead.error()) +/// } +/// } +/// } +/// ``` +#[macro_export(local_inner_macros)] +macro_rules! custom_keyword { + ($ident:ident) => { + #[allow(non_camel_case_types)] + pub struct $ident { + pub span: $crate::export::Span, + } + + #[doc(hidden)] + #[allow(non_snake_case)] + pub fn $ident<__S: $crate::export::IntoSpans<[$crate::export::Span; 1]>>( + span: __S, + ) -> $ident { + $ident { + span: $crate::export::IntoSpans::into_spans(span)[0], + } + } + + impl $crate::export::Default for $ident { + fn default() -> Self { + $ident { + span: $crate::export::Span::call_site(), + } + } + } + + impl_parse_for_custom_keyword!($ident); + impl_to_tokens_for_custom_keyword!($ident); + impl_clone_for_custom_keyword!($ident); + impl_extra_traits_for_custom_keyword!($ident); + }; +} + +// Not public API. +#[cfg(feature = "parsing")] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_parse_for_custom_keyword { + ($ident:ident) => { + // For peek. + impl $crate::token::CustomToken for $ident { + fn peek(cursor: $crate::buffer::Cursor) -> $crate::export::bool { + if let Some((ident, _rest)) = cursor.ident() { + ident == stringify!($ident) + } else { + false + } + } + + fn display() -> &'static $crate::export::str { + concat!("`", stringify!($ident), "`") + } + } + + impl $crate::parse::Parse for $ident { + fn parse(input: $crate::parse::ParseStream) -> $crate::parse::Result<$ident> { + input.step(|cursor| { + if let $crate::export::Some((ident, rest)) = cursor.ident() { + if ident == stringify!($ident) { + return $crate::export::Ok(($ident { span: ident.span() }, rest)); + } + } + $crate::export::Err(cursor.error(concat!( + "expected `", + stringify!($ident), + "`" + ))) + }) + } + } + }; +} + +// Not public API. +#[cfg(not(feature = "parsing"))] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_parse_for_custom_keyword { + ($ident:ident) => {}; +} + +// Not public API. +#[cfg(feature = "printing")] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_to_tokens_for_custom_keyword { + ($ident:ident) => { + impl $crate::export::ToTokens for $ident { + fn to_tokens(&self, tokens: &mut $crate::export::TokenStream2) { + let ident = $crate::Ident::new(stringify!($ident), self.span); + $crate::export::TokenStreamExt::append(tokens, ident); + } + } + }; +} + +// Not public API. +#[cfg(not(feature = "printing"))] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_to_tokens_for_custom_keyword { + ($ident:ident) => {}; +} + +// Not public API. +#[cfg(feature = "clone-impls")] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_clone_for_custom_keyword { + ($ident:ident) => { + impl $crate::export::Copy for $ident {} + + impl $crate::export::Clone for $ident { + fn clone(&self) -> Self { + *self + } + } + }; +} + +// Not public API. +#[cfg(not(feature = "clone-impls"))] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_clone_for_custom_keyword { + ($ident:ident) => {}; +} + +// Not public API. +#[cfg(feature = "extra-traits")] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_extra_traits_for_custom_keyword { + ($ident:ident) => { + impl $crate::export::Debug for $ident { + fn fmt(&self, f: &mut $crate::export::Formatter) -> $crate::export::fmt::Result { + $crate::export::Formatter::write_str( + f, + concat!("Keyword [", stringify!($ident), "]"), + ) + } + } + + impl $crate::export::Eq for $ident {} + + impl $crate::export::PartialEq for $ident { + fn eq(&self, _other: &Self) -> $crate::export::bool { + true + } + } + + impl $crate::export::Hash for $ident { + fn hash<__H: $crate::export::Hasher>(&self, _state: &mut __H) {} + } + }; +} + +// Not public API. +#[cfg(not(feature = "extra-traits"))] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_extra_traits_for_custom_keyword { + ($ident:ident) => {}; +} diff --git a/syn/src/custom_punctuation.rs b/syn/src/custom_punctuation.rs new file mode 100644 index 0000000..29fa448 --- /dev/null +++ b/syn/src/custom_punctuation.rs @@ -0,0 +1,309 @@ +/// Define a type that supports parsing and printing a multi-character symbol +/// as if it were a punctuation token. +/// +/// # Usage +/// +/// ``` +/// syn::custom_punctuation!(LeftRightArrow, <=>); +/// ``` +/// +/// The generated syntax tree node supports the following operations just like +/// any built-in punctuation token. +/// +/// - [Peeking] — `input.peek(LeftRightArrow)` +/// +/// - [Parsing] — `input.parse::()?` +/// +/// - [Printing] — `quote!( ... #lrarrow ... )` +/// +/// - Construction from a [`Span`] — `let lrarrow = LeftRightArrow(sp)` +/// +/// - Construction from multiple [`Span`] — `let lrarrow = LeftRightArrow([sp, sp, sp])` +/// +/// - Field access to its spans — `let spans = lrarrow.spans` +/// +/// [Peeking]: parse::ParseBuffer::peek +/// [Parsing]: parse::ParseBuffer::parse +/// [Printing]: quote::ToTokens +/// [`Span`]: proc_macro2::Span +/// +/// # Example +/// +/// ``` +/// use proc_macro2::{TokenStream, TokenTree}; +/// use syn::parse::{Parse, ParseStream, Peek, Result}; +/// use syn::punctuated::Punctuated; +/// use syn::Expr; +/// +/// syn::custom_punctuation!(PathSeparator, ); +/// +/// // expr expr expr ... +/// struct PathSegments { +/// segments: Punctuated, +/// } +/// +/// impl Parse for PathSegments { +/// fn parse(input: ParseStream) -> Result { +/// let mut segments = Punctuated::new(); +/// +/// let first = parse_until(input, PathSeparator)?; +/// segments.push_value(syn::parse2(first)?); +/// +/// while input.peek(PathSeparator) { +/// segments.push_punct(input.parse()?); +/// +/// let next = parse_until(input, PathSeparator)?; +/// segments.push_value(syn::parse2(next)?); +/// } +/// +/// Ok(PathSegments { segments }) +/// } +/// } +/// +/// fn parse_until(input: ParseStream, end: E) -> Result { +/// let mut tokens = TokenStream::new(); +/// while !input.is_empty() && !input.peek(end) { +/// let next: TokenTree = input.parse()?; +/// tokens.extend(Some(next)); +/// } +/// Ok(tokens) +/// } +/// +/// fn main() { +/// let input = r#" a::b c::d::e "#; +/// let _: PathSegments = syn::parse_str(input).unwrap(); +/// } +/// ``` +#[macro_export(local_inner_macros)] +macro_rules! custom_punctuation { + ($ident:ident, $($tt:tt)+) => { + pub struct $ident { + pub spans: custom_punctuation_repr!($($tt)+), + } + + #[doc(hidden)] + #[allow(non_snake_case)] + pub fn $ident<__S: $crate::export::IntoSpans>( + spans: __S, + ) -> $ident { + let _validate_len = 0 $(+ custom_punctuation_len!(strict, $tt))*; + $ident { + spans: $crate::export::IntoSpans::into_spans(spans) + } + } + + impl $crate::export::Default for $ident { + fn default() -> Self { + $ident($crate::export::Span::call_site()) + } + } + + impl_parse_for_custom_punctuation!($ident, $($tt)+); + impl_to_tokens_for_custom_punctuation!($ident, $($tt)+); + impl_clone_for_custom_punctuation!($ident, $($tt)+); + impl_extra_traits_for_custom_punctuation!($ident, $($tt)+); + }; +} + +// Not public API. +#[cfg(feature = "parsing")] +#[doc(hidden)] +#[macro_export(local_inner_macros)] +macro_rules! impl_parse_for_custom_punctuation { + ($ident:ident, $($tt:tt)+) => { + impl $crate::token::CustomToken for $ident { + fn peek(cursor: $crate::buffer::Cursor) -> bool { + $crate::token::parsing::peek_punct(cursor, stringify_punct!($($tt)+)) + } + + fn display() -> &'static $crate::export::str { + custom_punctuation_concat!("`", stringify_punct!($($tt)+), "`") + } + } + + impl $crate::parse::Parse for $ident { + fn parse(input: $crate::parse::ParseStream) -> $crate::parse::Result<$ident> { + let spans: custom_punctuation_repr!($($tt)+) = + $crate::token::parsing::punct(input, stringify_punct!($($tt)+))?; + Ok($ident(spans)) + } + } + }; +} + +// Not public API. +#[cfg(not(feature = "parsing"))] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_parse_for_custom_punctuation { + ($ident:ident, $($tt:tt)+) => {}; +} + +// Not public API. +#[cfg(feature = "printing")] +#[doc(hidden)] +#[macro_export(local_inner_macros)] +macro_rules! impl_to_tokens_for_custom_punctuation { + ($ident:ident, $($tt:tt)+) => { + impl $crate::export::ToTokens for $ident { + fn to_tokens(&self, tokens: &mut $crate::export::TokenStream2) { + $crate::token::printing::punct(stringify_punct!($($tt)+), &self.spans, tokens) + } + } + }; +} + +// Not public API. +#[cfg(not(feature = "printing"))] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_to_tokens_for_custom_punctuation { + ($ident:ident, $($tt:tt)+) => {}; +} + +// Not public API. +#[cfg(feature = "clone-impls")] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_clone_for_custom_punctuation { + ($ident:ident, $($tt:tt)+) => { + impl $crate::export::Copy for $ident {} + + impl $crate::export::Clone for $ident { + fn clone(&self) -> Self { + *self + } + } + }; +} + +// Not public API. +#[cfg(not(feature = "clone-impls"))] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_clone_for_custom_punctuation { + ($ident:ident, $($tt:tt)+) => {}; +} + +// Not public API. +#[cfg(feature = "extra-traits")] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_extra_traits_for_custom_punctuation { + ($ident:ident, $($tt:tt)+) => { + impl $crate::export::Debug for $ident { + fn fmt(&self, f: &mut $crate::export::Formatter) -> $crate::export::fmt::Result { + $crate::export::Formatter::write_str(f, stringify!($ident)) + } + } + + impl $crate::export::Eq for $ident {} + + impl $crate::export::PartialEq for $ident { + fn eq(&self, _other: &Self) -> $crate::export::bool { + true + } + } + + impl $crate::export::Hash for $ident { + fn hash<__H: $crate::export::Hasher>(&self, _state: &mut __H) {} + } + }; +} + +// Not public API. +#[cfg(not(feature = "extra-traits"))] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_extra_traits_for_custom_punctuation { + ($ident:ident, $($tt:tt)+) => {}; +} + +// Not public API. +#[doc(hidden)] +#[macro_export(local_inner_macros)] +macro_rules! custom_punctuation_repr { + ($($tt:tt)+) => { + [$crate::export::Span; 0 $(+ custom_punctuation_len!(lenient, $tt))+] + }; +} + +// Not public API. +#[doc(hidden)] +#[macro_export(local_inner_macros)] +#[rustfmt::skip] +macro_rules! custom_punctuation_len { + ($mode:ident, +) => { 1 }; + ($mode:ident, +=) => { 2 }; + ($mode:ident, &) => { 1 }; + ($mode:ident, &&) => { 2 }; + ($mode:ident, &=) => { 2 }; + ($mode:ident, @) => { 1 }; + ($mode:ident, !) => { 1 }; + ($mode:ident, ^) => { 1 }; + ($mode:ident, ^=) => { 2 }; + ($mode:ident, :) => { 1 }; + ($mode:ident, ::) => { 2 }; + ($mode:ident, ,) => { 1 }; + ($mode:ident, /) => { 1 }; + ($mode:ident, /=) => { 2 }; + ($mode:ident, .) => { 1 }; + ($mode:ident, ..) => { 2 }; + ($mode:ident, ...) => { 3 }; + ($mode:ident, ..=) => { 3 }; + ($mode:ident, =) => { 1 }; + ($mode:ident, ==) => { 2 }; + ($mode:ident, >=) => { 2 }; + ($mode:ident, >) => { 1 }; + ($mode:ident, <=) => { 2 }; + ($mode:ident, <) => { 1 }; + ($mode:ident, *=) => { 2 }; + ($mode:ident, !=) => { 2 }; + ($mode:ident, |) => { 1 }; + ($mode:ident, |=) => { 2 }; + ($mode:ident, ||) => { 2 }; + ($mode:ident, #) => { 1 }; + ($mode:ident, ?) => { 1 }; + ($mode:ident, ->) => { 2 }; + ($mode:ident, <-) => { 2 }; + ($mode:ident, %) => { 1 }; + ($mode:ident, %=) => { 2 }; + ($mode:ident, =>) => { 2 }; + ($mode:ident, ;) => { 1 }; + ($mode:ident, <<) => { 2 }; + ($mode:ident, <<=) => { 3 }; + ($mode:ident, >>) => { 2 }; + ($mode:ident, >>=) => { 3 }; + ($mode:ident, *) => { 1 }; + ($mode:ident, -) => { 1 }; + ($mode:ident, -=) => { 2 }; + ($mode:ident, ~) => { 1 }; + (lenient, $tt:tt) => { 0 }; + (strict, $tt:tt) => {{ custom_punctuation_unexpected!($tt); 0 }}; +} + +// Not public API. +#[doc(hidden)] +#[macro_export] +macro_rules! custom_punctuation_unexpected { + () => {}; +} + +// Not public API. +#[doc(hidden)] +#[macro_export] +macro_rules! stringify_punct { + ($($tt:tt)+) => { + concat!($(stringify!($tt)),+) + }; +} + +// Not public API. +// Without this, local_inner_macros breaks when looking for concat! +#[doc(hidden)] +#[macro_export] +macro_rules! custom_punctuation_concat { + ($($tt:tt)*) => { + concat!($($tt)*) + }; +} diff --git a/syn/src/data.rs b/syn/src/data.rs new file mode 100644 index 0000000..184a79e --- /dev/null +++ b/syn/src/data.rs @@ -0,0 +1,456 @@ +use super::*; +use crate::punctuated::Punctuated; + +ast_struct! { + /// An enum variant. + /// + /// *This type is available if Syn is built with the `"derive"` or `"full"` + /// feature.* + pub struct Variant { + /// Attributes tagged on the variant. + pub attrs: Vec, + + /// Name of the variant. + pub ident: Ident, + + /// Content stored in the variant. + pub fields: Fields, + + /// Explicit discriminant: `Variant = 1` + pub discriminant: Option<(Token![=], Expr)>, + } +} + +ast_enum_of_structs! { + /// Data stored within an enum variant or struct. + /// + /// *This type is available if Syn is built with the `"derive"` or `"full"` + /// feature.* + /// + /// # Syntax tree enum + /// + /// This type is a [syntax tree enum]. + /// + /// [syntax tree enum]: enum.Expr.html#syntax-tree-enums + // + // TODO: change syntax-tree-enum link to an intra rustdoc link, currently + // blocked on https://github.com/rust-lang/rust/issues/62833 + pub enum Fields { + /// Named fields of a struct or struct variant such as `Point { x: f64, + /// y: f64 }`. + Named(FieldsNamed), + + /// Unnamed fields of a tuple struct or tuple variant such as `Some(T)`. + Unnamed(FieldsUnnamed), + + /// Unit struct or unit variant such as `None`. + Unit, + } +} + +ast_struct! { + /// Named fields of a struct or struct variant such as `Point { x: f64, + /// y: f64 }`. + /// + /// *This type is available if Syn is built with the `"derive"` or + /// `"full"` feature.* + pub struct FieldsNamed { + pub brace_token: token::Brace, + pub named: Punctuated, + } +} + +ast_struct! { + /// Unnamed fields of a tuple struct or tuple variant such as `Some(T)`. + /// + /// *This type is available if Syn is built with the `"derive"` or + /// `"full"` feature.* + pub struct FieldsUnnamed { + pub paren_token: token::Paren, + pub unnamed: Punctuated, + } +} + +impl Fields { + /// Get an iterator over the borrowed [`Field`] items in this object. This + /// iterator can be used to iterate over a named or unnamed struct or + /// variant's fields uniformly. + pub fn iter(&self) -> punctuated::Iter { + match self { + Fields::Unit => crate::punctuated::empty_punctuated_iter(), + Fields::Named(f) => f.named.iter(), + Fields::Unnamed(f) => f.unnamed.iter(), + } + } + + /// Get an iterator over the mutably borrowed [`Field`] items in this + /// object. This iterator can be used to iterate over a named or unnamed + /// struct or variant's fields uniformly. + pub fn iter_mut(&mut self) -> punctuated::IterMut { + match self { + Fields::Unit => crate::punctuated::empty_punctuated_iter_mut(), + Fields::Named(f) => f.named.iter_mut(), + Fields::Unnamed(f) => f.unnamed.iter_mut(), + } + } + + /// Returns the number of fields. + pub fn len(&self) -> usize { + match self { + Fields::Unit => 0, + Fields::Named(f) => f.named.len(), + Fields::Unnamed(f) => f.unnamed.len(), + } + } + + /// Returns `true` if there are zero fields. + pub fn is_empty(&self) -> bool { + match self { + Fields::Unit => true, + Fields::Named(f) => f.named.is_empty(), + Fields::Unnamed(f) => f.unnamed.is_empty(), + } + } +} + +impl IntoIterator for Fields { + type Item = Field; + type IntoIter = punctuated::IntoIter; + + fn into_iter(self) -> Self::IntoIter { + match self { + Fields::Unit => Punctuated::::new().into_iter(), + Fields::Named(f) => f.named.into_iter(), + Fields::Unnamed(f) => f.unnamed.into_iter(), + } + } +} + +impl<'a> IntoIterator for &'a Fields { + type Item = &'a Field; + type IntoIter = punctuated::Iter<'a, Field>; + + fn into_iter(self) -> Self::IntoIter { + self.iter() + } +} + +impl<'a> IntoIterator for &'a mut Fields { + type Item = &'a mut Field; + type IntoIter = punctuated::IterMut<'a, Field>; + + fn into_iter(self) -> Self::IntoIter { + self.iter_mut() + } +} + +ast_struct! { + /// A field of a struct or enum variant. + /// + /// *This type is available if Syn is built with the `"derive"` or `"full"` + /// feature.* + pub struct Field { + /// Attributes tagged on the field. + pub attrs: Vec, + + /// Visibility of the field. + pub vis: Visibility, + + /// Name of the field, if any. + /// + /// Fields of tuple structs have no names. + pub ident: Option, + + pub colon_token: Option, + + /// Type of the field. + pub ty: Type, + } +} + +ast_enum_of_structs! { + /// The visibility level of an item: inherited or `pub` or + /// `pub(restricted)`. + /// + /// *This type is available if Syn is built with the `"derive"` or `"full"` + /// feature.* + /// + /// # Syntax tree enum + /// + /// This type is a [syntax tree enum]. + /// + /// [syntax tree enum]: enum.Expr.html#syntax-tree-enums + // + // TODO: change syntax-tree-enum link to an intra rustdoc link, currently + // blocked on https://github.com/rust-lang/rust/issues/62833 + pub enum Visibility { + /// A public visibility level: `pub`. + Public(VisPublic), + + /// A crate-level visibility: `crate`. + Crate(VisCrate), + + /// A visibility level restricted to some path: `pub(self)` or + /// `pub(super)` or `pub(crate)` or `pub(in some::module)`. + Restricted(VisRestricted), + + /// An inherited visibility, which usually means private. + Inherited, + } +} + +ast_struct! { + /// A public visibility level: `pub`. + /// + /// *This type is available if Syn is built with the `"derive"` or + /// `"full"` feature.* + pub struct VisPublic { + pub pub_token: Token![pub], + } +} + +ast_struct! { + /// A crate-level visibility: `crate`. + /// + /// *This type is available if Syn is built with the `"derive"` or + /// `"full"` feature.* + pub struct VisCrate { + pub crate_token: Token![crate], + } +} + +ast_struct! { + /// A visibility level restricted to some path: `pub(self)` or + /// `pub(super)` or `pub(crate)` or `pub(in some::module)`. + /// + /// *This type is available if Syn is built with the `"derive"` or + /// `"full"` feature.* + pub struct VisRestricted { + pub pub_token: Token![pub], + pub paren_token: token::Paren, + pub in_token: Option, + pub path: Box, + } +} + +#[cfg(feature = "parsing")] +pub mod parsing { + use super::*; + + use crate::ext::IdentExt; + use crate::parse::discouraged::Speculative; + use crate::parse::{Parse, ParseStream, Result}; + + impl Parse for Variant { + fn parse(input: ParseStream) -> Result { + Ok(Variant { + attrs: input.call(Attribute::parse_outer)?, + ident: input.parse()?, + fields: { + if input.peek(token::Brace) { + Fields::Named(input.parse()?) + } else if input.peek(token::Paren) { + Fields::Unnamed(input.parse()?) + } else { + Fields::Unit + } + }, + discriminant: { + if input.peek(Token![=]) { + let eq_token: Token![=] = input.parse()?; + let discriminant: Expr = input.parse()?; + Some((eq_token, discriminant)) + } else { + None + } + }, + }) + } + } + + impl Parse for FieldsNamed { + fn parse(input: ParseStream) -> Result { + let content; + Ok(FieldsNamed { + brace_token: braced!(content in input), + named: content.parse_terminated(Field::parse_named)?, + }) + } + } + + impl Parse for FieldsUnnamed { + fn parse(input: ParseStream) -> Result { + let content; + Ok(FieldsUnnamed { + paren_token: parenthesized!(content in input), + unnamed: content.parse_terminated(Field::parse_unnamed)?, + }) + } + } + + impl Field { + /// Parses a named (braced struct) field. + pub fn parse_named(input: ParseStream) -> Result { + Ok(Field { + attrs: input.call(Attribute::parse_outer)?, + vis: input.parse()?, + ident: Some(input.parse()?), + colon_token: Some(input.parse()?), + ty: input.parse()?, + }) + } + + /// Parses an unnamed (tuple struct) field. + pub fn parse_unnamed(input: ParseStream) -> Result { + Ok(Field { + attrs: input.call(Attribute::parse_outer)?, + vis: input.parse()?, + ident: None, + colon_token: None, + ty: input.parse()?, + }) + } + } + + impl Parse for Visibility { + fn parse(input: ParseStream) -> Result { + if input.peek(Token![pub]) { + Self::parse_pub(input) + } else if input.peek(Token![crate]) { + Self::parse_crate(input) + } else { + Ok(Visibility::Inherited) + } + } + } + + impl Visibility { + fn parse_pub(input: ParseStream) -> Result { + let pub_token = input.parse::()?; + + if input.peek(token::Paren) { + let ahead = input.fork(); + + let content; + let paren_token = parenthesized!(content in ahead); + if content.peek(Token![crate]) + || content.peek(Token![self]) + || content.peek(Token![super]) + { + let path = content.call(Ident::parse_any)?; + + // Ensure there are no additional tokens within `content`. + // Without explicitly checking, we may misinterpret a tuple + // field as a restricted visibility, causing a parse error. + // e.g. `pub (crate::A, crate::B)` (Issue #720). + if content.is_empty() { + input.advance_to(&ahead); + return Ok(Visibility::Restricted(VisRestricted { + pub_token, + paren_token, + in_token: None, + path: Box::new(Path::from(path)), + })); + } + } else if content.peek(Token![in]) { + let in_token: Token![in] = content.parse()?; + let path = content.call(Path::parse_mod_style)?; + + input.advance_to(&ahead); + return Ok(Visibility::Restricted(VisRestricted { + pub_token, + paren_token, + in_token: Some(in_token), + path: Box::new(path), + })); + } + } + + Ok(Visibility::Public(VisPublic { pub_token })) + } + + fn parse_crate(input: ParseStream) -> Result { + if input.peek2(Token![::]) { + Ok(Visibility::Inherited) + } else { + Ok(Visibility::Crate(VisCrate { + crate_token: input.parse()?, + })) + } + } + } +} + +#[cfg(feature = "printing")] +mod printing { + use super::*; + + use proc_macro2::TokenStream; + use quote::{ToTokens, TokenStreamExt}; + + use crate::print::TokensOrDefault; + + impl ToTokens for Variant { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append_all(&self.attrs); + self.ident.to_tokens(tokens); + self.fields.to_tokens(tokens); + if let Some((eq_token, disc)) = &self.discriminant { + eq_token.to_tokens(tokens); + disc.to_tokens(tokens); + } + } + } + + impl ToTokens for FieldsNamed { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.brace_token.surround(tokens, |tokens| { + self.named.to_tokens(tokens); + }); + } + } + + impl ToTokens for FieldsUnnamed { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.paren_token.surround(tokens, |tokens| { + self.unnamed.to_tokens(tokens); + }); + } + } + + impl ToTokens for Field { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append_all(&self.attrs); + self.vis.to_tokens(tokens); + if let Some(ident) = &self.ident { + ident.to_tokens(tokens); + TokensOrDefault(&self.colon_token).to_tokens(tokens); + } + self.ty.to_tokens(tokens); + } + } + + impl ToTokens for VisPublic { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.pub_token.to_tokens(tokens) + } + } + + impl ToTokens for VisCrate { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.crate_token.to_tokens(tokens); + } + } + + impl ToTokens for VisRestricted { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.pub_token.to_tokens(tokens); + self.paren_token.surround(tokens, |tokens| { + // TODO: If we have a path which is not "self" or "super" or + // "crate", automatically add the "in" token. + self.in_token.to_tokens(tokens); + self.path.to_tokens(tokens); + }); + } + } +} diff --git a/syn/src/derive.rs b/syn/src/derive.rs new file mode 100644 index 0000000..8cb9cf7 --- /dev/null +++ b/syn/src/derive.rs @@ -0,0 +1,273 @@ +use super::*; +use crate::punctuated::Punctuated; + +ast_struct! { + /// Data structure sent to a `proc_macro_derive` macro. + /// + /// *This type is available if Syn is built with the `"derive"` feature.* + pub struct DeriveInput { + /// Attributes tagged on the whole struct or enum. + pub attrs: Vec, + + /// Visibility of the struct or enum. + pub vis: Visibility, + + /// Name of the struct or enum. + pub ident: Ident, + + /// Generics required to complete the definition. + pub generics: Generics, + + /// Data within the struct or enum. + pub data: Data, + } +} + +ast_enum_of_structs! { + /// The storage of a struct, enum or union data structure. + /// + /// *This type is available if Syn is built with the `"derive"` feature.* + /// + /// # Syntax tree enum + /// + /// This type is a [syntax tree enum]. + /// + /// [syntax tree enum]: enum.Expr.html#syntax-tree-enums + // + // TODO: change syntax-tree-enum link to an intra rustdoc link, currently + // blocked on https://github.com/rust-lang/rust/issues/62833 + pub enum Data { + /// A struct input to a `proc_macro_derive` macro. + Struct(DataStruct), + + /// An enum input to a `proc_macro_derive` macro. + Enum(DataEnum), + + /// An untagged union input to a `proc_macro_derive` macro. + Union(DataUnion), + } + + do_not_generate_to_tokens +} + +ast_struct! { + /// A struct input to a `proc_macro_derive` macro. + /// + /// *This type is available if Syn is built with the `"derive"` + /// feature.* + pub struct DataStruct { + pub struct_token: Token![struct], + pub fields: Fields, + pub semi_token: Option, + } +} + +ast_struct! { + /// An enum input to a `proc_macro_derive` macro. + /// + /// *This type is available if Syn is built with the `"derive"` + /// feature.* + pub struct DataEnum { + pub enum_token: Token![enum], + pub brace_token: token::Brace, + pub variants: Punctuated, + } +} + +ast_struct! { + /// An untagged union input to a `proc_macro_derive` macro. + /// + /// *This type is available if Syn is built with the `"derive"` + /// feature.* + pub struct DataUnion { + pub union_token: Token![union], + pub fields: FieldsNamed, + } +} + +#[cfg(feature = "parsing")] +pub mod parsing { + use super::*; + + use crate::parse::{Parse, ParseStream, Result}; + + impl Parse for DeriveInput { + fn parse(input: ParseStream) -> Result { + let attrs = input.call(Attribute::parse_outer)?; + let vis = input.parse::()?; + + let lookahead = input.lookahead1(); + if lookahead.peek(Token![struct]) { + let struct_token = input.parse::()?; + let ident = input.parse::()?; + let generics = input.parse::()?; + let (where_clause, fields, semi) = data_struct(input)?; + Ok(DeriveInput { + attrs, + vis, + ident, + generics: Generics { + where_clause, + ..generics + }, + data: Data::Struct(DataStruct { + struct_token, + fields, + semi_token: semi, + }), + }) + } else if lookahead.peek(Token![enum]) { + let enum_token = input.parse::()?; + let ident = input.parse::()?; + let generics = input.parse::()?; + let (where_clause, brace, variants) = data_enum(input)?; + Ok(DeriveInput { + attrs, + vis, + ident, + generics: Generics { + where_clause, + ..generics + }, + data: Data::Enum(DataEnum { + enum_token, + brace_token: brace, + variants, + }), + }) + } else if lookahead.peek(Token![union]) { + let union_token = input.parse::()?; + let ident = input.parse::()?; + let generics = input.parse::()?; + let (where_clause, fields) = data_union(input)?; + Ok(DeriveInput { + attrs, + vis, + ident, + generics: Generics { + where_clause, + ..generics + }, + data: Data::Union(DataUnion { + union_token, + fields, + }), + }) + } else { + Err(lookahead.error()) + } + } + } + + pub fn data_struct( + input: ParseStream, + ) -> Result<(Option, Fields, Option)> { + let mut lookahead = input.lookahead1(); + let mut where_clause = None; + if lookahead.peek(Token![where]) { + where_clause = Some(input.parse()?); + lookahead = input.lookahead1(); + } + + if where_clause.is_none() && lookahead.peek(token::Paren) { + let fields = input.parse()?; + + lookahead = input.lookahead1(); + if lookahead.peek(Token![where]) { + where_clause = Some(input.parse()?); + lookahead = input.lookahead1(); + } + + if lookahead.peek(Token![;]) { + let semi = input.parse()?; + Ok((where_clause, Fields::Unnamed(fields), Some(semi))) + } else { + Err(lookahead.error()) + } + } else if lookahead.peek(token::Brace) { + let fields = input.parse()?; + Ok((where_clause, Fields::Named(fields), None)) + } else if lookahead.peek(Token![;]) { + let semi = input.parse()?; + Ok((where_clause, Fields::Unit, Some(semi))) + } else { + Err(lookahead.error()) + } + } + + pub fn data_enum( + input: ParseStream, + ) -> Result<( + Option, + token::Brace, + Punctuated, + )> { + let where_clause = input.parse()?; + + let content; + let brace = braced!(content in input); + let variants = content.parse_terminated(Variant::parse)?; + + Ok((where_clause, brace, variants)) + } + + pub fn data_union(input: ParseStream) -> Result<(Option, FieldsNamed)> { + let where_clause = input.parse()?; + let fields = input.parse()?; + Ok((where_clause, fields)) + } +} + +#[cfg(feature = "printing")] +mod printing { + use super::*; + + use proc_macro2::TokenStream; + use quote::ToTokens; + + use crate::attr::FilterAttrs; + use crate::print::TokensOrDefault; + + impl ToTokens for DeriveInput { + fn to_tokens(&self, tokens: &mut TokenStream) { + for attr in self.attrs.outer() { + attr.to_tokens(tokens); + } + self.vis.to_tokens(tokens); + match &self.data { + Data::Struct(d) => d.struct_token.to_tokens(tokens), + Data::Enum(d) => d.enum_token.to_tokens(tokens), + Data::Union(d) => d.union_token.to_tokens(tokens), + } + self.ident.to_tokens(tokens); + self.generics.to_tokens(tokens); + match &self.data { + Data::Struct(data) => match &data.fields { + Fields::Named(fields) => { + self.generics.where_clause.to_tokens(tokens); + fields.to_tokens(tokens); + } + Fields::Unnamed(fields) => { + fields.to_tokens(tokens); + self.generics.where_clause.to_tokens(tokens); + TokensOrDefault(&data.semi_token).to_tokens(tokens); + } + Fields::Unit => { + self.generics.where_clause.to_tokens(tokens); + TokensOrDefault(&data.semi_token).to_tokens(tokens); + } + }, + Data::Enum(data) => { + self.generics.where_clause.to_tokens(tokens); + data.brace_token.surround(tokens, |tokens| { + data.variants.to_tokens(tokens); + }); + } + Data::Union(data) => { + self.generics.where_clause.to_tokens(tokens); + data.fields.to_tokens(tokens); + } + } + } + } +} diff --git a/syn/src/discouraged.rs b/syn/src/discouraged.rs new file mode 100644 index 0000000..29d1006 --- /dev/null +++ b/syn/src/discouraged.rs @@ -0,0 +1,195 @@ +//! Extensions to the parsing API with niche applicability. + +use super::*; + +/// Extensions to the `ParseStream` API to support speculative parsing. +pub trait Speculative { + /// Advance this parse stream to the position of a forked parse stream. + /// + /// This is the opposite operation to [`ParseStream::fork`]. You can fork a + /// parse stream, perform some speculative parsing, then join the original + /// stream to the fork to "commit" the parsing from the fork to the main + /// stream. + /// + /// If you can avoid doing this, you should, as it limits the ability to + /// generate useful errors. That said, it is often the only way to parse + /// syntax of the form `A* B*` for arbitrary syntax `A` and `B`. The problem + /// is that when the fork fails to parse an `A`, it's impossible to tell + /// whether that was because of a syntax error and the user meant to provide + /// an `A`, or that the `A`s are finished and its time to start parsing + /// `B`s. Use with care. + /// + /// Also note that if `A` is a subset of `B`, `A* B*` can be parsed by + /// parsing `B*` and removing the leading members of `A` from the + /// repetition, bypassing the need to involve the downsides associated with + /// speculative parsing. + /// + /// [`ParseStream::fork`]: ParseBuffer::fork + /// + /// # Example + /// + /// There has been chatter about the possibility of making the colons in the + /// turbofish syntax like `path::to::` no longer required by accepting + /// `path::to` in expression position. Specifically, according to [RFC + /// 2544], [`PathSegment`] parsing should always try to consume a following + /// `<` token as the start of generic arguments, and reset to the `<` if + /// that fails (e.g. the token is acting as a less-than operator). + /// + /// This is the exact kind of parsing behavior which requires the "fork, + /// try, commit" behavior that [`ParseStream::fork`] discourages. With + /// `advance_to`, we can avoid having to parse the speculatively parsed + /// content a second time. + /// + /// This change in behavior can be implemented in syn by replacing just the + /// `Parse` implementation for `PathSegment`: + /// + /// ``` + /// # use syn::ext::IdentExt; + /// use syn::parse::discouraged::Speculative; + /// # use syn::parse::{Parse, ParseStream}; + /// # use syn::{Ident, PathArguments, Result, Token}; + /// + /// pub struct PathSegment { + /// pub ident: Ident, + /// pub arguments: PathArguments, + /// } + /// # + /// # impl From for PathSegment + /// # where + /// # T: Into, + /// # { + /// # fn from(ident: T) -> Self { + /// # PathSegment { + /// # ident: ident.into(), + /// # arguments: PathArguments::None, + /// # } + /// # } + /// # } + /// + /// impl Parse for PathSegment { + /// fn parse(input: ParseStream) -> Result { + /// if input.peek(Token![super]) + /// || input.peek(Token![self]) + /// || input.peek(Token![Self]) + /// || input.peek(Token![crate]) + /// || input.peek(Token![extern]) + /// { + /// let ident = input.call(Ident::parse_any)?; + /// return Ok(PathSegment::from(ident)); + /// } + /// + /// let ident = input.parse()?; + /// if input.peek(Token![::]) && input.peek3(Token![<]) { + /// return Ok(PathSegment { + /// ident, + /// arguments: PathArguments::AngleBracketed(input.parse()?), + /// }); + /// } + /// if input.peek(Token![<]) && !input.peek(Token![<=]) { + /// let fork = input.fork(); + /// if let Ok(arguments) = fork.parse() { + /// input.advance_to(&fork); + /// return Ok(PathSegment { + /// ident, + /// arguments: PathArguments::AngleBracketed(arguments), + /// }); + /// } + /// } + /// Ok(PathSegment::from(ident)) + /// } + /// } + /// + /// # syn::parse_str::("a").unwrap(); + /// ``` + /// + /// # Drawbacks + /// + /// The main drawback of this style of speculative parsing is in error + /// presentation. Even if the lookahead is the "correct" parse, the error + /// that is shown is that of the "fallback" parse. To use the same example + /// as the turbofish above, take the following unfinished "turbofish": + /// + /// ```text + /// let _ = f<&'a fn(), for<'a> serde::>(); + /// ``` + /// + /// If this is parsed as generic arguments, we can provide the error message + /// + /// ```text + /// error: expected identifier + /// --> src.rs:L:C + /// | + /// L | let _ = f<&'a fn(), for<'a> serde::>(); + /// | ^ + /// ``` + /// + /// but if parsed using the above speculative parsing, it falls back to + /// assuming that the `<` is a less-than when it fails to parse the generic + /// arguments, and tries to interpret the `&'a` as the start of a labelled + /// loop, resulting in the much less helpful error + /// + /// ```text + /// error: expected `:` + /// --> src.rs:L:C + /// | + /// L | let _ = f<&'a fn(), for<'a> serde::>(); + /// | ^^ + /// ``` + /// + /// This can be mitigated with various heuristics (two examples: show both + /// forks' parse errors, or show the one that consumed more tokens), but + /// when you can control the grammar, sticking to something that can be + /// parsed LL(3) and without the LL(*) speculative parsing this makes + /// possible, displaying reasonable errors becomes much more simple. + /// + /// [RFC 2544]: https://github.com/rust-lang/rfcs/pull/2544 + /// [`PathSegment`]: crate::PathSegment + /// + /// # Performance + /// + /// This method performs a cheap fixed amount of work that does not depend + /// on how far apart the two streams are positioned. + /// + /// # Panics + /// + /// The forked stream in the argument of `advance_to` must have been + /// obtained by forking `self`. Attempting to advance to any other stream + /// will cause a panic. + fn advance_to(&self, fork: &Self); +} + +impl<'a> Speculative for ParseBuffer<'a> { + fn advance_to(&self, fork: &Self) { + if !crate::buffer::same_scope(self.cursor(), fork.cursor()) { + panic!("Fork was not derived from the advancing parse stream"); + } + + let (self_unexp, self_sp) = inner_unexpected(self); + let (fork_unexp, fork_sp) = inner_unexpected(fork); + if !Rc::ptr_eq(&self_unexp, &fork_unexp) { + match (fork_sp, self_sp) { + // Unexpected set on the fork, but not on `self`, copy it over. + (Some(span), None) => { + self_unexp.set(Unexpected::Some(span)); + } + // Unexpected unset. Use chain to propagate errors from fork. + (None, None) => { + fork_unexp.set(Unexpected::Chain(self_unexp)); + + // Ensure toplevel 'unexpected' tokens from the fork don't + // bubble up the chain by replacing the root `unexpected` + // pointer, only 'unexpected' tokens from existing group + // parsers should bubble. + fork.unexpected + .set(Some(Rc::new(Cell::new(Unexpected::None)))); + } + // Unexpected has been set on `self`. No changes needed. + (_, Some(_)) => {} + } + } + + // See comment on `cell` in the struct definition. + self.cell + .set(unsafe { mem::transmute::>(fork.cursor()) }) + } +} diff --git a/syn/src/error.rs b/syn/src/error.rs new file mode 100644 index 0000000..146d652 --- /dev/null +++ b/syn/src/error.rs @@ -0,0 +1,357 @@ +use std; +use std::fmt::{self, Debug, Display}; +use std::iter::FromIterator; +use std::slice; +use std::vec; + +use proc_macro2::{ + Delimiter, Group, Ident, LexError, Literal, Punct, Spacing, Span, TokenStream, TokenTree, +}; +#[cfg(feature = "printing")] +use quote::ToTokens; + +#[cfg(feature = "parsing")] +use crate::buffer::Cursor; +use crate::thread::ThreadBound; + +/// The result of a Syn parser. +pub type Result = std::result::Result; + +/// Error returned when a Syn parser cannot parse the input tokens. +/// +/// # Error reporting in proc macros +/// +/// The correct way to report errors back to the compiler from a procedural +/// macro is by emitting an appropriately spanned invocation of +/// [`compile_error!`] in the generated code. This produces a better diagnostic +/// message than simply panicking the macro. +/// +/// [`compile_error!`]: https://doc.rust-lang.org/std/macro.compile_error.html +/// +/// When parsing macro input, the [`parse_macro_input!`] macro handles the +/// conversion to `compile_error!` automatically. +/// +/// ``` +/// extern crate proc_macro; +/// +/// use proc_macro::TokenStream; +/// use syn::{parse_macro_input, AttributeArgs, ItemFn}; +/// +/// # const IGNORE: &str = stringify! { +/// #[proc_macro_attribute] +/// # }; +/// pub fn my_attr(args: TokenStream, input: TokenStream) -> TokenStream { +/// let args = parse_macro_input!(args as AttributeArgs); +/// let input = parse_macro_input!(input as ItemFn); +/// +/// /* ... */ +/// # TokenStream::new() +/// } +/// ``` +/// +/// For errors that arise later than the initial parsing stage, the +/// [`.to_compile_error()`] method can be used to perform an explicit conversion +/// to `compile_error!`. +/// +/// [`.to_compile_error()`]: Error::to_compile_error +/// +/// ``` +/// # extern crate proc_macro; +/// # +/// # use proc_macro::TokenStream; +/// # use syn::{parse_macro_input, DeriveInput}; +/// # +/// # const IGNORE: &str = stringify! { +/// #[proc_macro_derive(MyDerive)] +/// # }; +/// pub fn my_derive(input: TokenStream) -> TokenStream { +/// let input = parse_macro_input!(input as DeriveInput); +/// +/// // fn(DeriveInput) -> syn::Result +/// expand::my_derive(input) +/// .unwrap_or_else(|err| err.to_compile_error()) +/// .into() +/// } +/// # +/// # mod expand { +/// # use proc_macro2::TokenStream; +/// # use syn::{DeriveInput, Result}; +/// # +/// # pub fn my_derive(input: DeriveInput) -> Result { +/// # unimplemented!() +/// # } +/// # } +/// ``` +#[derive(Clone)] +pub struct Error { + messages: Vec, +} + +struct ErrorMessage { + // Span is implemented as an index into a thread-local interner to keep the + // size small. It is not safe to access from a different thread. We want + // errors to be Send and Sync to play nicely with the Failure crate, so pin + // the span we're given to its original thread and assume it is + // Span::call_site if accessed from any other thread. + start_span: ThreadBound, + end_span: ThreadBound, + message: String, +} + +#[cfg(test)] +struct _Test +where + Error: Send + Sync; + +impl Error { + /// Usually the [`ParseStream::error`] method will be used instead, which + /// automatically uses the correct span from the current position of the + /// parse stream. + /// + /// Use `Error::new` when the error needs to be triggered on some span other + /// than where the parse stream is currently positioned. + /// + /// [`ParseStream::error`]: crate::parse::ParseBuffer::error + /// + /// # Example + /// + /// ``` + /// use syn::{Error, Ident, LitStr, Result, Token}; + /// use syn::parse::ParseStream; + /// + /// // Parses input that looks like `name = "string"` where the key must be + /// // the identifier `name` and the value may be any string literal. + /// // Returns the string literal. + /// fn parse_name(input: ParseStream) -> Result { + /// let name_token: Ident = input.parse()?; + /// if name_token != "name" { + /// // Trigger an error not on the current position of the stream, + /// // but on the position of the unexpected identifier. + /// return Err(Error::new(name_token.span(), "expected `name`")); + /// } + /// input.parse::()?; + /// let s: LitStr = input.parse()?; + /// Ok(s) + /// } + /// ``` + pub fn new(span: Span, message: T) -> Self { + Error { + messages: vec![ErrorMessage { + start_span: ThreadBound::new(span), + end_span: ThreadBound::new(span), + message: message.to_string(), + }], + } + } + + /// Creates an error with the specified message spanning the given syntax + /// tree node. + /// + /// Unlike the `Error::new` constructor, this constructor takes an argument + /// `tokens` which is a syntax tree node. This allows the resulting `Error` + /// to attempt to span all tokens inside of `tokens`. While you would + /// typically be able to use the `Spanned` trait with the above `Error::new` + /// constructor, implementation limitations today mean that + /// `Error::new_spanned` may provide a higher-quality error message on + /// stable Rust. + /// + /// When in doubt it's recommended to stick to `Error::new` (or + /// `ParseStream::error`)! + #[cfg(feature = "printing")] + pub fn new_spanned(tokens: T, message: U) -> Self { + let mut iter = tokens.into_token_stream().into_iter(); + let start = iter.next().map_or_else(Span::call_site, |t| t.span()); + let end = iter.last().map_or(start, |t| t.span()); + Error { + messages: vec![ErrorMessage { + start_span: ThreadBound::new(start), + end_span: ThreadBound::new(end), + message: message.to_string(), + }], + } + } + + /// The source location of the error. + /// + /// Spans are not thread-safe so this function returns `Span::call_site()` + /// if called from a different thread than the one on which the `Error` was + /// originally created. + pub fn span(&self) -> Span { + let start = match self.messages[0].start_span.get() { + Some(span) => *span, + None => return Span::call_site(), + }; + let end = match self.messages[0].end_span.get() { + Some(span) => *span, + None => return Span::call_site(), + }; + start.join(end).unwrap_or(start) + } + + /// Render the error as an invocation of [`compile_error!`]. + /// + /// The [`parse_macro_input!`] macro provides a convenient way to invoke + /// this method correctly in a procedural macro. + /// + /// [`compile_error!`]: https://doc.rust-lang.org/std/macro.compile_error.html + pub fn to_compile_error(&self) -> TokenStream { + self.messages + .iter() + .map(ErrorMessage::to_compile_error) + .collect() + } + + /// Add another error message to self such that when `to_compile_error()` is + /// called, both errors will be emitted together. + pub fn combine(&mut self, another: Error) { + self.messages.extend(another.messages) + } +} + +impl ErrorMessage { + fn to_compile_error(&self) -> TokenStream { + let start = self + .start_span + .get() + .cloned() + .unwrap_or_else(Span::call_site); + let end = self.end_span.get().cloned().unwrap_or_else(Span::call_site); + + // compile_error!($message) + TokenStream::from_iter(vec![ + TokenTree::Ident(Ident::new("compile_error", start)), + TokenTree::Punct({ + let mut punct = Punct::new('!', Spacing::Alone); + punct.set_span(start); + punct + }), + TokenTree::Group({ + let mut group = Group::new(Delimiter::Brace, { + TokenStream::from_iter(vec![TokenTree::Literal({ + let mut string = Literal::string(&self.message); + string.set_span(end); + string + })]) + }); + group.set_span(end); + group + }), + ]) + } +} + +#[cfg(feature = "parsing")] +pub fn new_at(scope: Span, cursor: Cursor, message: T) -> Error { + if cursor.eof() { + Error::new(scope, format!("unexpected end of input, {}", message)) + } else { + let span = crate::buffer::open_span_of_group(cursor); + Error::new(span, message) + } +} + +impl Debug for Error { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + if self.messages.len() == 1 { + formatter + .debug_tuple("Error") + .field(&self.messages[0]) + .finish() + } else { + formatter + .debug_tuple("Error") + .field(&self.messages) + .finish() + } + } +} + +impl Debug for ErrorMessage { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + Debug::fmt(&self.message, formatter) + } +} + +impl Display for Error { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str(&self.messages[0].message) + } +} + +impl Clone for ErrorMessage { + fn clone(&self) -> Self { + let start = self + .start_span + .get() + .cloned() + .unwrap_or_else(Span::call_site); + let end = self.end_span.get().cloned().unwrap_or_else(Span::call_site); + ErrorMessage { + start_span: ThreadBound::new(start), + end_span: ThreadBound::new(end), + message: self.message.clone(), + } + } +} + +impl std::error::Error for Error { + fn description(&self) -> &str { + "parse error" + } +} + +impl From for Error { + fn from(err: LexError) -> Self { + Error::new(Span::call_site(), format!("{:?}", err)) + } +} + +impl IntoIterator for Error { + type Item = Error; + type IntoIter = IntoIter; + + fn into_iter(self) -> Self::IntoIter { + IntoIter { + messages: self.messages.into_iter(), + } + } +} + +pub struct IntoIter { + messages: vec::IntoIter, +} + +impl Iterator for IntoIter { + type Item = Error; + + fn next(&mut self) -> Option { + Some(Error { + messages: vec![self.messages.next()?], + }) + } +} + +impl<'a> IntoIterator for &'a Error { + type Item = Error; + type IntoIter = Iter<'a>; + + fn into_iter(self) -> Self::IntoIter { + Iter { + messages: self.messages.iter(), + } + } +} + +pub struct Iter<'a> { + messages: slice::Iter<'a, ErrorMessage>, +} + +impl<'a> Iterator for Iter<'a> { + type Item = Error; + + fn next(&mut self) -> Option { + Some(Error { + messages: vec![self.messages.next()?.clone()], + }) + } +} diff --git a/syn/src/export.rs b/syn/src/export.rs new file mode 100644 index 0000000..37dc467 --- /dev/null +++ b/syn/src/export.rs @@ -0,0 +1,35 @@ +pub use std::clone::Clone; +pub use std::cmp::{Eq, PartialEq}; +pub use std::convert::From; +pub use std::default::Default; +pub use std::fmt::{self, Debug, Formatter}; +pub use std::hash::{Hash, Hasher}; +pub use std::marker::Copy; +pub use std::option::Option::{None, Some}; +pub use std::result::Result::{Err, Ok}; + +#[cfg(feature = "printing")] +pub extern crate quote; + +pub use proc_macro2::{Span, TokenStream as TokenStream2}; + +pub use crate::span::IntoSpans; + +#[cfg(all( + not(all(target_arch = "wasm32", any(target_os = "unknown", target_os = "wasi"))), + feature = "proc-macro" +))] +pub use proc_macro::TokenStream; + +#[cfg(feature = "printing")] +pub use quote::{ToTokens, TokenStreamExt}; + +#[allow(non_camel_case_types)] +pub type bool = help::Bool; +#[allow(non_camel_case_types)] +pub type str = help::Str; + +mod help { + pub type Bool = bool; + pub type Str = str; +} diff --git a/syn/src/expr.rs b/syn/src/expr.rs new file mode 100644 index 0000000..603dc32 --- /dev/null +++ b/syn/src/expr.rs @@ -0,0 +1,3236 @@ +use super::*; +use crate::punctuated::Punctuated; +#[cfg(feature = "extra-traits")] +use crate::tt::TokenStreamHelper; +use proc_macro2::{Span, TokenStream}; +#[cfg(feature = "printing")] +use quote::IdentFragment; +#[cfg(feature = "printing")] +use std::fmt::{self, Display}; +use std::hash::{Hash, Hasher}; +#[cfg(all(feature = "parsing", feature = "full"))] +use std::mem; + +ast_enum_of_structs! { + /// A Rust expression. + /// + /// *This type is available if Syn is built with the `"derive"` or `"full"` + /// feature.* + /// + /// # Syntax tree enums + /// + /// This type is a syntax tree enum. In Syn this and other syntax tree enums + /// are designed to be traversed using the following rebinding idiom. + /// + /// ``` + /// # use syn::Expr; + /// # + /// # fn example(expr: Expr) { + /// # const IGNORE: &str = stringify! { + /// let expr: Expr = /* ... */; + /// # }; + /// match expr { + /// Expr::MethodCall(expr) => { + /// /* ... */ + /// } + /// Expr::Cast(expr) => { + /// /* ... */ + /// } + /// Expr::If(expr) => { + /// /* ... */ + /// } + /// + /// /* ... */ + /// # _ => {} + /// # } + /// # } + /// ``` + /// + /// We begin with a variable `expr` of type `Expr` that has no fields + /// (because it is an enum), and by matching on it and rebinding a variable + /// with the same name `expr` we effectively imbue our variable with all of + /// the data fields provided by the variant that it turned out to be. So for + /// example above if we ended up in the `MethodCall` case then we get to use + /// `expr.receiver`, `expr.args` etc; if we ended up in the `If` case we get + /// to use `expr.cond`, `expr.then_branch`, `expr.else_branch`. + /// + /// This approach avoids repeating the variant names twice on every line. + /// + /// ``` + /// # use syn::{Expr, ExprMethodCall}; + /// # + /// # fn example(expr: Expr) { + /// // Repetitive; recommend not doing this. + /// match expr { + /// Expr::MethodCall(ExprMethodCall { method, args, .. }) => { + /// # } + /// # _ => {} + /// # } + /// # } + /// ``` + /// + /// In general, the name to which a syntax tree enum variant is bound should + /// be a suitable name for the complete syntax tree enum type. + /// + /// ``` + /// # use syn::{Expr, ExprField}; + /// # + /// # fn example(discriminant: ExprField) { + /// // Binding is called `base` which is the name I would use if I were + /// // assigning `*discriminant.base` without an `if let`. + /// if let Expr::Tuple(base) = *discriminant.base { + /// # } + /// # } + /// ``` + /// + /// A sign that you may not be choosing the right variable names is if you + /// see names getting repeated in your code, like accessing + /// `receiver.receiver` or `pat.pat` or `cond.cond`. + pub enum Expr #manual_extra_traits { + /// A slice literal expression: `[a, b, c, d]`. + Array(ExprArray), + + /// An assignment expression: `a = compute()`. + Assign(ExprAssign), + + /// A compound assignment expression: `counter += 1`. + AssignOp(ExprAssignOp), + + /// An async block: `async { ... }`. + Async(ExprAsync), + + /// An await expression: `fut.await`. + Await(ExprAwait), + + /// A binary operation: `a + b`, `a * b`. + Binary(ExprBinary), + + /// A blocked scope: `{ ... }`. + Block(ExprBlock), + + /// A box expression: `box f`. + Box(ExprBox), + + /// A `break`, with an optional label to break and an optional + /// expression. + Break(ExprBreak), + + /// A function call expression: `invoke(a, b)`. + Call(ExprCall), + + /// A cast expression: `foo as f64`. + Cast(ExprCast), + + /// A closure expression: `|a, b| a + b`. + Closure(ExprClosure), + + /// A `continue`, with an optional label. + Continue(ExprContinue), + + /// Access of a named struct field (`obj.k`) or unnamed tuple struct + /// field (`obj.0`). + Field(ExprField), + + /// A for loop: `for pat in expr { ... }`. + ForLoop(ExprForLoop), + + /// An expression contained within invisible delimiters. + /// + /// This variant is important for faithfully representing the precedence + /// of expressions and is related to `None`-delimited spans in a + /// `TokenStream`. + Group(ExprGroup), + + /// An `if` expression with an optional `else` block: `if expr { ... } + /// else { ... }`. + /// + /// The `else` branch expression may only be an `If` or `Block` + /// expression, not any of the other types of expression. + If(ExprIf), + + /// A square bracketed indexing expression: `vector[2]`. + Index(ExprIndex), + + /// A `let` guard: `let Some(x) = opt`. + Let(ExprLet), + + /// A literal in place of an expression: `1`, `"foo"`. + Lit(ExprLit), + + /// Conditionless loop: `loop { ... }`. + Loop(ExprLoop), + + /// A macro invocation expression: `format!("{}", q)`. + Macro(ExprMacro), + + /// A `match` expression: `match n { Some(n) => {}, None => {} }`. + Match(ExprMatch), + + /// A method call expression: `x.foo::(a, b)`. + MethodCall(ExprMethodCall), + + /// A parenthesized expression: `(a + b)`. + Paren(ExprParen), + + /// A path like `std::mem::replace` possibly containing generic + /// parameters and a qualified self-type. + /// + /// A plain identifier like `x` is a path of length 1. + Path(ExprPath), + + /// A range expression: `1..2`, `1..`, `..2`, `1..=2`, `..=2`. + Range(ExprRange), + + /// A referencing operation: `&a` or `&mut a`. + Reference(ExprReference), + + /// An array literal constructed from one repeated element: `[0u8; N]`. + Repeat(ExprRepeat), + + /// A `return`, with an optional value to be returned. + Return(ExprReturn), + + /// A struct literal expression: `Point { x: 1, y: 1 }`. + /// + /// The `rest` provides the value of the remaining fields as in `S { a: + /// 1, b: 1, ..rest }`. + Struct(ExprStruct), + + /// A try-expression: `expr?`. + Try(ExprTry), + + /// A try block: `try { ... }`. + TryBlock(ExprTryBlock), + + /// A tuple expression: `(a, b, c, d)`. + Tuple(ExprTuple), + + /// A type ascription expression: `foo: f64`. + Type(ExprType), + + /// A unary operation: `!x`, `*x`. + Unary(ExprUnary), + + /// An unsafe block: `unsafe { ... }`. + Unsafe(ExprUnsafe), + + /// Tokens in expression position not interpreted by Syn. + Verbatim(TokenStream), + + /// A while loop: `while expr { ... }`. + While(ExprWhile), + + /// A yield expression: `yield expr`. + Yield(ExprYield), + + #[doc(hidden)] + __Nonexhaustive, + } +} + +ast_struct! { + /// A slice literal expression: `[a, b, c, d]`. + /// + /// *This type is available if Syn is built with the `"full"` feature.* + pub struct ExprArray #full { + pub attrs: Vec, + pub bracket_token: token::Bracket, + pub elems: Punctuated, + } +} + +ast_struct! { + /// An assignment expression: `a = compute()`. + /// + /// *This type is available if Syn is built with the `"full"` feature.* + pub struct ExprAssign #full { + pub attrs: Vec, + pub left: Box, + pub eq_token: Token![=], + pub right: Box, + } +} + +ast_struct! { + /// A compound assignment expression: `counter += 1`. + /// + /// *This type is available if Syn is built with the `"full"` feature.* + pub struct ExprAssignOp #full { + pub attrs: Vec, + pub left: Box, + pub op: BinOp, + pub right: Box, + } +} + +ast_struct! { + /// An async block: `async { ... }`. + /// + /// *This type is available if Syn is built with the `"full"` feature.* + pub struct ExprAsync #full { + pub attrs: Vec, + pub async_token: Token![async], + pub capture: Option, + pub block: Block, + } +} + +ast_struct! { + /// An await expression: `fut.await`. + /// + /// *This type is available if Syn is built with the `"full"` feature.* + pub struct ExprAwait #full { + pub attrs: Vec, + pub base: Box, + pub dot_token: Token![.], + pub await_token: token::Await, + } +} + +ast_struct! { + /// A binary operation: `a + b`, `a * b`. + /// + /// *This type is available if Syn is built with the `"derive"` or + /// `"full"` feature.* + pub struct ExprBinary { + pub attrs: Vec, + pub left: Box, + pub op: BinOp, + pub right: Box, + } +} + +ast_struct! { + /// A blocked scope: `{ ... }`. + /// + /// *This type is available if Syn is built with the `"full"` feature.* + pub struct ExprBlock #full { + pub attrs: Vec, + pub label: Option