restore newly-confluence #check_simp

closes #4452

.github/workflows/ci.yml

@@ -122,9 +122,8 @@ jobs:
           script: |
             const level = ${{ steps.set-level.outputs.check-level }};
             console.log(`level: ${level}`);
-            // use large runners outside PRs where available (original repo)
-            // disabled for now as this mostly just speeds up the test suite which is not a bottleneck
-            // let large = ${{ github.event_name != 'pull_request' && github.repository == 'leanprover/lean4' }} ? "-large" : "";
+            // use large runners where available (original repo)
+            let large = ${{ github.repository == 'leanprover/lean4' }};
             let matrix = [
               {
                 // portable release build: use channel with older glibc (2.27)
@@ -143,7 +142,7 @@ jobs:
               },
               {
                 "name": "Linux release",
-                "os": "ubuntu-latest",
+                "os": large ? "nscloud-ubuntu-22.04-amd64-4x8" : "ubuntu-latest",
                 "release": true,
                 "check-level": 0,
                 "shell": "nix develop .#oldGlibc -c bash -euxo pipefail {0}",
@@ -155,7 +154,7 @@ jobs:
               },
               {
                 "name": "Linux",
-                "os": "ubuntu-latest",
+                "os": large ? "nscloud-ubuntu-22.04-amd64-4x8" : "ubuntu-latest",
                 "check-stage3": level >= 2,
                 "test-speedcenter": level >= 2,
                 "check-level": 1,
@@ -281,16 +280,8 @@ jobs:
       CXX: c++
       MACOSX_DEPLOYMENT_TARGET: 10.15
     steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-        with:
-          submodules: true
-          # the default is to use a virtual merge commit between the PR and master: just use the PR
-          ref: ${{ github.event.pull_request.head.sha }}
       - name: Install Nix
-        uses: cachix/install-nix-action@v18
-        with:
-          install_url: https://releases.nixos.org/nix/nix-2.12.0/install
+        uses: DeterminateSystems/nix-installer-action@main
         if: runner.os == 'Linux' && !matrix.cmultilib
       - name: Install MSYS2
         uses: msys2/setup-msys2@v2
@@ -303,6 +294,20 @@ jobs:
         run: |
           brew install ccache tree zstd coreutils gmp
         if: runner.os == 'macOS'
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          # the default is to use a virtual merge commit between the PR and master: just use the PR
+          ref: ${{ github.event.pull_request.head.sha }}
+      # Do check out some CI-relevant files from virtual merge commit to accommodate CI changes on
+      # master (as the workflow files themselves are always taken from the merge)
+      # (needs to be after "Install *" to use the right shell)
+      - name: CI Merge Checkout
+        run: |
+          git fetch --depth=1 origin ${{ github.sha }}
+          git checkout FETCH_HEAD flake.nix flake.lock
+        if: github.event_name == 'pull_request'
+      # (needs to be after "Checkout" so files don't get overriden)
       - name: Setup emsdk
         uses: mymindstorm/setup-emsdk@v12
         with:
@@ -318,20 +323,14 @@ jobs:
         uses: actions/cache@v3
         with:
           path: .ccache
-          key: ${{ matrix.name }}-build-v3-${{ github.sha }}
+          key: ${{ matrix.name }}-build-v3-${{ github.event.pull_request.head.sha }}
           # fall back to (latest) previous cache
           restore-keys: |
             ${{ matrix.name }}-build-v3
+      # open nix-shell once for initial setup
       - name: Setup
         run: |
-          # open nix-shell once for initial setup
-          true
-        if: runner.os == 'Linux'
-      - name: Set up core dumps
-        run: |
-          mkdir -p $PWD/coredumps
-          # store in current directory, for easy uploading together with binary
-          echo $PWD/coredumps/%e.%p.%t | sudo tee /proc/sys/kernel/core_pattern
+          ccache --zero-stats
         if: runner.os == 'Linux'
       - name: Set up NPROC
         run: |
@@ -340,7 +339,6 @@ jobs:
         run: |
           mkdir build
           cd build
-          ulimit -c unlimited # coredumps
           # arguments passed to `cmake`
           # this also enables githash embedding into stage 1 library
           OPTIONS=(-DCHECK_OLEAN_VERSION=ON)
@@ -367,8 +365,10 @@ jobs:
           fi
           # contortion to support empty OPTIONS with old macOS bash
           cmake .. --preset ${{ matrix.CMAKE_PRESET || 'release' }} -B . ${{ matrix.CMAKE_OPTIONS }} ${OPTIONS[@]+"${OPTIONS[@]}"} -DLEAN_INSTALL_PREFIX=$PWD/..
-          make -j$NPROC
-          make install
+          time make -j$NPROC
+      - name: Install
+        run: |
+          make -C build install
       - name: Check Binaries
         run: ${{ matrix.binary-check }} lean-*/bin/* || true
       - name: List Install Tree
@@ -398,8 +398,7 @@ jobs:
       - name: Test
         id: test
         run: |
-          ulimit -c unlimited # coredumps
-          ctest --preset ${{ matrix.CMAKE_PRESET || 'release' }} --test-dir build/stage1 -j$NPROC --output-junit test-results.xml ${{ matrix.CTEST_OPTIONS }}
+          time ctest --preset ${{ matrix.CMAKE_PRESET || 'release' }} --test-dir build/stage1 -j$NPROC --output-junit test-results.xml ${{ matrix.CTEST_OPTIONS }}
         if: (matrix.wasm || !matrix.cross) && needs.configure.outputs.check-level >= 1
       - name: Test Summary
         uses: test-summary/action@v2
@@ -412,51 +411,28 @@ jobs:
         if: (!matrix.cross) && steps.test.conclusion != 'skipped'
       - name: Build Stage 2
         run: |
-          ulimit -c unlimited # coredumps
           make -C build -j$NPROC stage2
         if: matrix.test-speedcenter
       - name: Check Stage 3
         run: |
-          ulimit -c unlimited # coredumps
           make -C build -j$NPROC stage3
         if: matrix.test-speedcenter
       - name: Test Speedcenter Benchmarks
         run: |
-          echo -1 | sudo tee /proc/sys/kernel/perf_event_paranoid
+          # Necessary for some timing metrics but does not work on Namespace runners
+          # and we just want to test that the benchmarks run at all here
+          #echo -1 | sudo tee /proc/sys/kernel/perf_event_paranoid
           export BUILD=$PWD/build PATH=$PWD/build/stage1/bin:$PATH
           cd tests/bench
           nix shell .#temci -c temci exec --config speedcenter.yaml --included_blocks fast --runs 1
         if: matrix.test-speedcenter
       - name: Check rebootstrap
         run: |
-          ulimit -c unlimited # coredumps
           # clean rebuild in case of Makefile changes
           make -C build update-stage0 && rm -rf build/stage* && make -C build -j$NPROC
         if: matrix.name == 'Linux' && needs.configure.outputs.check-level >= 1
       - name: CCache stats
         run: ccache -s
-      - name: Show stacktrace for coredumps
-        if: ${{ failure() && runner.os == 'Linux' }}
-        run: |
-          for c in coredumps/*; do
-            progbin="$(file $c | sed "s/.*execfn: '\([^']*\)'.*/\1/")"
-            echo bt | $GDB/bin/gdb -q $progbin $c || true
-          done
-      # has not been used in a long while, would need to be adapted to new
-      # shared libs
-      #- name: Upload coredumps
-      #  uses: actions/upload-artifact@v3
-      #  if: ${{ failure() && runner.os == 'Linux' }}
-      #  with:
-      #    name: coredumps-${{ matrix.name }}
-      #    path: |
-      #      ./coredumps
-      #      ./build/stage0/bin/lean
-      #      ./build/stage0/lib/lean/libleanshared.so
-      #      ./build/stage1/bin/lean
-      #      ./build/stage1/lib/lean/libleanshared.so
-      #      ./build/stage2/bin/lean
-      #      ./build/stage2/lib/lean/libleanshared.so
 
   # This job collects results from all the matrix jobs
   # This can be made the “required” job, instead of listing each

.github/workflows/nix-ci.yml

@@ -13,18 +13,36 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  # see ci.yml
+  configure:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.result }}
+    steps:
+      - name: Configure build matrix
+        id: set-matrix
+        uses: actions/github-script@v7
+        with:
+          script: |
+            let large = ${{ github.repository == 'leanprover/lean4' }};
+            let matrix = [
+              {
+                "name": "Nix Linux",
+                "os": large ? "nscloud-ubuntu-22.04-amd64-8x8" : "ubuntu-latest",
+              }
+            ];
+            console.log(`matrix:\n${JSON.stringify(matrix, null, 2)}`);
+            return matrix;
+
   Build:
+    needs: [configure]
     runs-on: ${{ matrix.os }}
     defaults:
       run:
         shell: nix run .#ciShell -- bash -euxo pipefail {0}
     strategy:
       matrix:
-        include:
-          - name: Nix Linux
-            os: ubuntu-latest
-          #- name: Nix macOS
-          #  os: macos-latest
+        include: ${{fromJson(needs.configure.outputs.matrix)}}
       # complete all jobs
       fail-fast: false
     name: ${{ matrix.name }}

.github/workflows/pr-release.yml

@@ -298,6 +298,13 @@ jobs:
           ref: nightly-testing
           fetch-depth: 0 # This ensures we check out all tags and branches.
 
+      - name: install elan
+        run: |
+          set -o pipefail
+          curl -sSfL https://github.com/leanprover/elan/releases/download/v3.0.0/elan-x86_64-unknown-linux-gnu.tar.gz | tar xz
+          ./elan-init -y --default-toolchain none
+          echo "$HOME/.elan/bin" >> "${GITHUB_PATH}"
+
       - name: Check if tag exists
         if: steps.workflow-info.outputs.pullRequestNumber != '' && steps.ready.outputs.mathlib_ready == 'true'
         id: check_mathlib_tag
@@ -322,7 +329,8 @@ jobs:
             echo "leanprover/lean4-pr-releases:pr-release-${{ steps.workflow-info.outputs.pullRequestNumber }}" > lean-toolchain
             git add lean-toolchain
             sed -i "s/require batteries from git \"https:\/\/github.com\/leanprover-community\/batteries\" @ \".\+\"/require batteries from git \"https:\/\/github.com\/leanprover-community\/batteries\" @ \"nightly-testing-${MOST_RECENT_NIGHTLY}\"/" lakefile.lean
-            git add lakefile.lean
+            lake update batteries
+            git add lakefile.lean lake-manifest.json
             git commit -m "Update lean-toolchain for testing https://github.com/leanprover/lean4/pull/${{ steps.workflow-info.outputs.pullRequestNumber }}"
           else
             echo "Branch already exists, pushing an empty commit."

.gitignore

@@ -4,8 +4,10 @@
 *.lock
 .lake
 lake-manifest.json
-build
-!/src/lake/Lake/Build
+/build
+/src/lakefile.toml
+/tests/lakefile.toml
+/lakefile.toml
 GPATH
 GRTAGS
 GSYMS

RELEASES.md

@@ -1,23 +1,515 @@
 # Lean 4 releases
 
-This file contains release notes for each stable release.
-Please check the [releases](https://github.com/leanprover/lean4/releases) page for the current status
-of each version.
-During development, drafts of future release notes appear in [`releases_drafts`](https://github.com/leanprover/lean4/tree/master/script).
-
 We intend to provide regular "minor version" releases of the Lean language at approximately monthly intervals.
 There is not yet a strong guarantee of backwards compatibility between versions,
 only an expectation that breaking changes will be documented in this file.
 
-v4.9.0
----------
+This file contains work-in-progress notes for the upcoming release, as well as previous stable releases.
+Please check the [releases](https://github.com/leanprover/lean4/releases) page for the current status
+of each version.
 
+v4.10.0
+----------
 Development in progress.
 
+v4.9.0
+---------- 
+Release candidate, release notes will be copied from branch `releases/v4.9.0` once completed.
+
 v4.8.0
 ---------
 
-Release candidate, release notes will be copied from branch `releases/v4.8.0` once completed.
+### Language features, tactics, and metaprograms
+
+* **Functional induction principles.**
+  [#3432](https://github.com/leanprover/lean4/pull/3432), [#3620](https://github.com/leanprover/lean4/pull/3620),
+  [#3754](https://github.com/leanprover/lean4/pull/3754), [#3762](https://github.com/leanprover/lean4/pull/3762),
+  [#3738](https://github.com/leanprover/lean4/pull/3738), [#3776](https://github.com/leanprover/lean4/pull/3776),
+  [#3898](https://github.com/leanprover/lean4/pull/3898).
+
+  Derived from the definition of a (possibly mutually) recursive function,
+  a **functional induction principle** is created that is tailored to proofs about that function.
+
+  For example from:
+  ```
+  def ackermann : Nat → Nat → Nat
+    | 0, m => m + 1
+    | n+1, 0 => ackermann n 1
+    | n+1, m+1 => ackermann n (ackermann (n + 1) m)
+  ```
+  we get
+  ```
+  ackermann.induct (motive : Nat → Nat → Prop) (case1 : ∀ (m : Nat), motive 0 m)
+    (case2 : ∀ (n : Nat), motive n 1 → motive (Nat.succ n) 0)
+    (case3 : ∀ (n m : Nat), motive (n + 1) m → motive n (ackermann (n + 1) m) → motive (Nat.succ n) (Nat.succ m))
+    (x x : Nat) : motive x x
+  ```
+
+  It can be used in the `induction` tactic using the `using` syntax:
+  ```
+  induction n, m using ackermann.induct
+  ```
+* The termination checker now recognizes more recursion patterns without an
+  explicit `termination_by`. In particular the idiom of counting up to an upper
+  bound, as in
+  ```
+  def Array.sum (arr : Array Nat) (i acc : Nat) : Nat :=
+    if _ : i < arr.size then
+      Array.sum arr (i+1) (acc + arr[i])
+    else
+      acc
+  ```
+  is recognized without having to say `termination_by arr.size - i`.
+  * [#3630](https://github.com/leanprover/lean4/pull/3630) makes `termination_by?` not use `sizeOf` when not needed
+  * [#3652](https://github.com/leanprover/lean4/pull/3652) improves the `termination_by` syntax.
+  * [#3658](https://github.com/leanprover/lean4/pull/3658) changes how termination arguments are elaborated.
+  * [#3665](https://github.com/leanprover/lean4/pull/3665) refactors GuessLex to allow inferring more complex termination arguments
+  * [#3666](https://github.com/leanprover/lean4/pull/3666) infers termination arguments such as `xs.size - i`
+* [#3629](https://github.com/leanprover/lean4/pull/3629),
+  [#3655](https://github.com/leanprover/lean4/pull/3655),
+  [#3747](https://github.com/leanprover/lean4/pull/3747):
+  Adds `@[induction_eliminator]` and `@[cases_eliminator]` attributes to be able to define custom eliminators
+  for the `induction` and `cases` tactics, replacing the `@[eliminator]` attribute.
+  Gives custom eliminators for `Nat` so that `induction` and `cases` put goal states into terms of `0` and `n + 1`
+  rather than `Nat.zero` and `Nat.succ n`.
+  Added option `tactic.customEliminators` to control whether to use custom eliminators.
+  Added a hack for `rcases`/`rintro`/`obtain` to use the custom eliminator for `Nat`.
+* **Shorter instances names.** There is a new algorithm for generating names for anonymous instances.
+  Across Std and Mathlib, the median ratio between lengths of new names and of old names is about 72%.
+  With the old algorithm, the longest name was 1660 characters, and now the longest name is 202 characters.
+  The new algorithm's 95th percentile name length is 67 characters, versus 278 for the old algorithm.
+  While the new algorithm produces names that are 1.2% less unique,
+  it avoids cross-project collisions by adding a module-based suffix
+  when it does not refer to declarations from the same "project" (modules that share the same root).
+  [#3089](https://github.com/leanprover/lean4/pull/3089)
+  and [#3934](https://github.com/leanprover/lean4/pull/3934).
+* [8d2adf](https://github.com/leanprover/lean4/commit/8d2adf521d2b7636347a5b01bfe473bf0fcfaf31)
+  Importing two different files containing proofs of the same theorem is no longer considered an error.
+  This feature is particularly useful for theorems that are automatically generated on demand (e.g., equational theorems).
+* [84b091](https://github.com/leanprover/lean4/commit/84b0919a116e9be12f933e764474f45d964ce85c)
+  Lean now generates an error if the type of a theorem is **not** a proposition.
+* **Definition transparency.** [47a343](https://github.com/leanprover/lean4/commit/47a34316fc03ce936fddd2d3dce44784c5bcdfa9). `@[reducible]`, `@[semireducible]`, and `@[irreducible]` are now scoped and able to be set for imported declarations.
+* `simp`/`dsimp`
+  * [#3607](https://github.com/leanprover/lean4/pull/3607) enables kernel projection reduction in `dsimp`
+  * [b24fbf](https://github.com/leanprover/lean4/commit/b24fbf44f3aaa112f5d799ef2a341772d1eb222d)
+    and [acdb00](https://github.com/leanprover/lean4/commit/acdb0054d5a0efa724cff596ac26852fad5724c4):
+    `dsimproc` command
+    to define defeq-preserving simplification procedures.
+  * [#3624](https://github.com/leanprover/lean4/pull/3624) makes `dsimp` normalize raw nat literals as `OfNat.ofNat` applications.
+  * [#3628](https://github.com/leanprover/lean4/pull/3628) makes `simp` correctly handle `OfScientific.ofScientific` literals.
+  * [#3654](https://github.com/leanprover/lean4/pull/3654) makes `dsimp?` report used simprocs.
+  * [dee074](https://github.com/leanprover/lean4/commit/dee074dcde03a37b7895a4901df2e4fa490c73c7) fixes equation theorem
+    handling in `simp` for non-recursive definitions.
+  * [#3819](https://github.com/leanprover/lean4/pull/3819) improved performance when simp encounters a loop.
+  * [#3821](https://github.com/leanprover/lean4/pull/3821) fixes discharger/cache interaction.
+  * [#3824](https://github.com/leanprover/lean4/pull/3824) keeps `simp` from breaking `Char` literals.
+  * [#3838](https://github.com/leanprover/lean4/pull/3838) allows `Nat` instances matching to be more lenient.
+  * [#3870](https://github.com/leanprover/lean4/pull/3870) documentation for `simp` configuration options.
+  * [#3972](https://github.com/leanprover/lean4/pull/3972) fixes simp caching.
+  * [#4044](https://github.com/leanprover/lean4/pull/4044) improves cache behavior for "well-behaved" dischargers.
+* `omega`
+  * [#3639](https://github.com/leanprover/lean4/pull/3639), [#3766](https://github.com/leanprover/lean4/pull/3766),
+    [#3853](https://github.com/leanprover/lean4/pull/3853), [#3875](https://github.com/leanprover/lean4/pull/3875):
+    introduces a term canonicalizer.
+  * [#3736](https://github.com/leanprover/lean4/pull/3736) improves handling of positivity for the modulo operator for `Int`.
+  * [#3828](https://github.com/leanprover/lean4/pull/3828) makes it work as a `simp` discharger.
+  * [#3847](https://github.com/leanprover/lean4/pull/3847) adds helpful error messages.
+* `rfl`
+  * [#3671](https://github.com/leanprover/lean4/pull/3671), [#3708](https://github.com/leanprover/lean4/pull/3708): upstreams the `@[refl]` attribute and the `rfl` tactic.
+  * [#3751](https://github.com/leanprover/lean4/pull/3751) makes `apply_rfl` not operate on `Eq` itself.
+  * [#4067](https://github.com/leanprover/lean4/pull/4067) improves error message when there are no goals.
+* [#3719](https://github.com/leanprover/lean4/pull/3719) upstreams the `rw?` tactic, with fixes and improvements in
+  [#3783](https://github.com/leanprover/lean4/pull/3783), [#3794](https://github.com/leanprover/lean4/pull/3794),
+  [#3911](https://github.com/leanprover/lean4/pull/3911).
+* `conv`
+  * [#3659](https://github.com/leanprover/lean4/pull/3659) adds a `conv` version of the `calc` tactic.
+  * [#3763](https://github.com/leanprover/lean4/pull/3763) makes `conv` clean up using `try with_reducible rfl` instead of `try rfl`.
+* `#guard_msgs`
+  * [#3617](https://github.com/leanprover/lean4/pull/3617) introduces whitespace protection using the `⏎` character.
+  * [#3883](https://github.com/leanprover/lean4/pull/3883):
+    The `#guard_msgs` command now has options to change whitespace normalization and sensitivity to message ordering.
+    For example, `#guard_msgs (whitespace := lax) in cmd` collapses whitespace before checking messages,
+    and `#guard_msgs (ordering := sorted) in cmd` sorts the messages in lexicographic order before checking.
+  * [#3931](https://github.com/leanprover/lean4/pull/3931) adds an unused variables ignore function for `#guard_msgs`.
+  * [#3912](https://github.com/leanprover/lean4/pull/3912) adds a diff between the expected and actual outputs. This feature is currently
+    disabled by default, but can be enabled with `set_option guard_msgs.diff true`.
+    Depending on user feedback, this option may default to `true` in a future version of Lean.
+* `do` **notation**
+  * [#3820](https://github.com/leanprover/lean4/pull/3820) makes it an error to lift `(<- ...)` out of a pure `if ... then ... else ...`
+* **Lazy discrimination trees**
+  * [#3610](https://github.com/leanprover/lean4/pull/3610) fixes a name collision for `LazyDiscrTree` that could lead to cache poisoning.
+  * [#3677](https://github.com/leanprover/lean4/pull/3677) simplifies and fixes `LazyDiscrTree` handling for `exact?`/`apply?`.
+  * [#3685](https://github.com/leanprover/lean4/pull/3685) moves general `exact?`/`apply?` functionality into `LazyDiscrTree`.
+  * [#3769](https://github.com/leanprover/lean4/pull/3769) has lemma selection improvements for `rw?` and `LazyDiscrTree`.
+  * [#3818](https://github.com/leanprover/lean4/pull/3818) improves ordering of matches.
+* [#3590](https://github.com/leanprover/lean4/pull/3590) adds `inductive.autoPromoteIndices` option to be able to disable auto promotion of indices in the `inductive` command.
+* **Miscellaneous bug fixes and improvements**
+  * [#3606](https://github.com/leanprover/lean4/pull/3606) preserves `cache` and `dischargeDepth` fields in `Lean.Meta.Simp.Result.mkEqSymm`.
+  * [#3633](https://github.com/leanprover/lean4/pull/3633) makes `elabTermEnsuringType` respect `errToSorry`, improving error recovery of the `have` tactic.
+  * [#3647](https://github.com/leanprover/lean4/pull/3647) enables `noncomputable unsafe` definitions, for deferring implementations until later.
+  * [#3672](https://github.com/leanprover/lean4/pull/3672) adjust namespaces of tactics.
+  * [#3725](https://github.com/leanprover/lean4/pull/3725) fixes `Ord` derive handler for indexed inductive types with unused alternatives.
+  * [#3893](https://github.com/leanprover/lean4/pull/3893) improves performance of derived `Ord` instances.
+  * [#3771](https://github.com/leanprover/lean4/pull/3771) changes error reporting for failing tactic macros. Improves `rfl` error message.
+  * [#3745](https://github.com/leanprover/lean4/pull/3745) fixes elaboration of generalized field notation if the object of the notation is an optional parameter.
+  * [#3799](https://github.com/leanprover/lean4/pull/3799) makes commands such as `universe`, `variable`, `namespace`, etc. require that their argument appear in a later column.
+    Commands that can optionally parse an `ident` or parse any number of `ident`s generally should require
+    that the `ident` use `colGt`. This keeps typos in commands from being interpreted as identifiers.
+  * [#3815](https://github.com/leanprover/lean4/pull/3815) lets the `split` tactic be used for writing code.
+  * [#3822](https://github.com/leanprover/lean4/pull/3822) adds missing info in `induction` tactic for `with` clauses of the form `| cstr a b c => ?_`.
+  * [#3806](https://github.com/leanprover/lean4/pull/3806) fixes `withSetOptionIn` combinator.
+  * [#3844](https://github.com/leanprover/lean4/pull/3844) removes unused `trace.Elab.syntax` option.
+  * [#3896](https://github.com/leanprover/lean4/pull/3896) improves hover and go-to-def for `attribute` command.
+  * [#3989](https://github.com/leanprover/lean4/pull/3989) makes linter options more discoverable.
+  * [#3916](https://github.com/leanprover/lean4/pull/3916) fixes go-to-def for syntax defined with `@[builtin_term_parser]`.
+  * [#3962](https://github.com/leanprover/lean4/pull/3962) fixes how `solveByElim` handles `symm` lemmas, making `exact?`/`apply?` usable again.
+  * [#3968](https://github.com/leanprover/lean4/pull/3968) improves the `@[deprecated]` attribute, adding `(since := "<date>")` field.
+  * [#3768](https://github.com/leanprover/lean4/pull/3768) makes `#print` command show structure fields.
+  * [#3974](https://github.com/leanprover/lean4/pull/3974) makes `exact?%` behave like `by exact?` rather than `by apply?`.
+  * [#3994](https://github.com/leanprover/lean4/pull/3994) makes elaboration of `he ▸ h` notation more predictable.
+  * [#3991](https://github.com/leanprover/lean4/pull/3991) adjusts transparency for `decreasing_trivial` macros.
+  * [#4092](https://github.com/leanprover/lean4/pull/4092) improves performance of `binop%` and `binrel%` expression tree elaborators.
+* **Docs:** [#3748](https://github.com/leanprover/lean4/pull/3748), [#3796](https://github.com/leanprover/lean4/pull/3796),
+[#3800](https://github.com/leanprover/lean4/pull/3800), [#3874](https://github.com/leanprover/lean4/pull/3874),
+[#3863](https://github.com/leanprover/lean4/pull/3863), [#3862](https://github.com/leanprover/lean4/pull/3862),
+[#3891](https://github.com/leanprover/lean4/pull/3891), [#3873](https://github.com/leanprover/lean4/pull/3873),
+[#3908](https://github.com/leanprover/lean4/pull/3908), [#3872](https://github.com/leanprover/lean4/pull/3872).
+
+### Language server and IDE extensions
+
+* [#3602](https://github.com/leanprover/lean4/pull/3602) enables `import` auto-completions.
+* [#3608](https://github.com/leanprover/lean4/pull/3608) fixes issue [leanprover/vscode-lean4#392](https://github.com/leanprover/vscode-lean4/issues/392).
+  Diagnostic ranges had an off-by-one error that would misplace goal states for example.
+* [#3014](https://github.com/leanprover/lean4/pull/3014) introduces snapshot trees, foundational work for incremental tactics and parallelism.
+  [#3849](https://github.com/leanprover/lean4/pull/3849) adds basic incrementality API.
+* [#3271](https://github.com/leanprover/lean4/pull/3271) adds support for server-to-client requests.
+* [#3656](https://github.com/leanprover/lean4/pull/3656) fixes jump to definition when there are conflicting names from different files.
+  Fixes issue [#1170](https://github.com/leanprover/lean4/issues/1170).
+* [#3691](https://github.com/leanprover/lean4/pull/3691), [#3925](https://github.com/leanprover/lean4/pull/3925),
+  [#3932](https://github.com/leanprover/lean4/pull/3932) keep semantic tokens synchronized (used for semantic highlighting), with performance improvements.
+* [#3247](https://github.com/leanprover/lean4/pull/3247) and [#3730](https://github.com/leanprover/lean4/pull/3730)
+  add diagnostics to run "Restart File" when a file dependency is saved.
+* [#3722](https://github.com/leanprover/lean4/pull/3722) uses the correct module names when displaying references.
+* [#3728](https://github.com/leanprover/lean4/pull/3728) makes errors in header reliably appear and makes the "Import out of date" warning be at "hint" severity.
+  [#3739](https://github.com/leanprover/lean4/pull/3739) simplifies the text of this warning.
+* [#3778](https://github.com/leanprover/lean4/pull/3778) fixes [#3462](https://github.com/leanprover/lean4/issues/3462),
+  where info nodes from before the cursor would be used for computing completions.
+* [#3985](https://github.com/leanprover/lean4/pull/3985) makes trace timings appear in Infoview.
+
+### Pretty printing
+
+* [#3797](https://github.com/leanprover/lean4/pull/3797) fixes the hovers over binders so that they show their types.
+* [#3640](https://github.com/leanprover/lean4/pull/3640) and [#3735](https://github.com/leanprover/lean4/pull/3735): Adds attribute `@[pp_using_anonymous_constructor]` to make structures pretty print as `⟨x, y, z⟩`
+  rather than as `{a := x, b := y, c := z}`.
+  This attribute is applied to `Sigma`, `PSigma`, `PProd`, `Subtype`, `And`, and `Fin`.
+* [#3749](https://github.com/leanprover/lean4/pull/3749)
+  Now structure instances pretty print with parent structures' fields inlined.
+  That is, if `B` extends `A`, then `{ toA := { x := 1 }, y := 2 }` now pretty prints as `{ x := 1, y := 2 }`.
+  Setting option `pp.structureInstances.flatten` to false turns this off.
+* [#3737](https://github.com/leanprover/lean4/pull/3737), [#3744](https://github.com/leanprover/lean4/pull/3744)
+  and [#3750](https://github.com/leanprover/lean4/pull/3750):
+  Option `pp.structureProjections` is renamed to `pp.fieldNotation`, and there is now a suboption `pp.fieldNotation.generalized`
+  to enable pretty printing function applications using generalized field notation (defaults to true).
+  Field notation can be disabled on a function-by-function basis using the `@[pp_nodot]` attribute.
+  The notation is not used for theorems.
+* [#4071](https://github.com/leanprover/lean4/pull/4071) fixes interaction between app unexpanders and `pp.fieldNotation.generalized`
+* [#3625](https://github.com/leanprover/lean4/pull/3625) makes `delabConstWithSignature` (used by `#check`) have the ability to put arguments "after the colon"
+  to avoid printing inaccessible names.
+* [#3798](https://github.com/leanprover/lean4/pull/3798),
+  [#3978](https://github.com/leanprover/lean4/pull/3978),
+  [#3798](https://github.com/leanprover/lean4/pull/3980):
+  Adds options `pp.mvars` (default: true) and `pp.mvars.withType` (default: false).
+  When `pp.mvars` is false, expression metavariables pretty print as `?_` and universe metavariables pretty print as `_`.
+  When `pp.mvars.withType` is true, expression metavariables pretty print with a type ascription.
+  These can be set when using `#guard_msgs` to make tests not depend on the particular names of metavariables.
+* [#3917](https://github.com/leanprover/lean4/pull/3917) makes binders hoverable and gives them docstrings.
+* [#4034](https://github.com/leanprover/lean4/pull/4034) makes hovers for RHS terms in `match` expressions in the Infoview reliably show the correct term.
+
+### Library
+
+* `Bool`/`Prop`
+  * [#3508](https://github.com/leanprover/lean4/pull/3508) improves `simp` confluence for `Bool` and `Prop` terms.
+  * Theorems: [#3604](https://github.com/leanprover/lean4/pull/3604)
+* `Nat`
+  * [#3579](https://github.com/leanprover/lean4/pull/3579) makes `Nat.succ_eq_add_one` be a simp lemma, now that `induction`/`cases` uses `n + 1` instead of `Nat.succ n`.
+  * [#3808](https://github.com/leanprover/lean4/pull/3808) replaces `Nat.succ` simp rules with simprocs.
+  * [#3876](https://github.com/leanprover/lean4/pull/3876) adds faster `Nat.repr` implementation in C.
+* `Int`
+  * Theorems: [#3890](https://github.com/leanprover/lean4/pull/3890)
+* `UInt`s
+  * [#3960](https://github.com/leanprover/lean4/pull/3960) improves performance of upcasting.
+* `Array` and `Subarray`
+  * [#3676](https://github.com/leanprover/lean4/pull/3676) removes `Array.eraseIdxAux`, `Array.eraseIdxSzAux`, and `Array.eraseIdx'`.
+  * [#3648](https://github.com/leanprover/lean4/pull/3648) simplifies `Array.findIdx?`.
+  * [#3851](https://github.com/leanprover/lean4/pull/3851) renames fields of `Subarray`.
+* `List`
+  * [#3785](https://github.com/leanprover/lean4/pull/3785) upstreams tail-recursive List operations and `@[csimp]` lemmas.
+* `BitVec`
+  * Theorems: [#3593](https://github.com/leanprover/lean4/pull/3593),
+  [#3593](https://github.com/leanprover/lean4/pull/3593), [#3597](https://github.com/leanprover/lean4/pull/3597),
+  [#3598](https://github.com/leanprover/lean4/pull/3598), [#3721](https://github.com/leanprover/lean4/pull/3721),
+  [#3729](https://github.com/leanprover/lean4/pull/3729), [#3880](https://github.com/leanprover/lean4/pull/3880),
+  [#4039](https://github.com/leanprover/lean4/pull/4039).
+  * [#3884](https://github.com/leanprover/lean4/pull/3884) protects `Std.BitVec`.
+* `String`
+  * [#3832](https://github.com/leanprover/lean4/pull/3832) fixes `String.splitOn`.
+  * [#3959](https://github.com/leanprover/lean4/pull/3959) adds `String.Pos.isValid`.
+  * [#3959](https://github.com/leanprover/lean4/pull/3959) UTF-8 string validation.
+  * [#3961](https://github.com/leanprover/lean4/pull/3961) adds a model implementation for UTF-8 encoding and decoding.
+* `IO`
+  * [#4097](https://github.com/leanprover/lean4/pull/4097) adds `IO.getTaskState` which returns whether a task is finished, actively running, or waiting on other Tasks to finish.
+
+* **Refactors**
+  * [#3605](https://github.com/leanprover/lean4/pull/3605) reduces imports for `Init.Data.Nat` and `Init.Data.Int`.
+  * [#3613](https://github.com/leanprover/lean4/pull/3613) reduces imports for `Init.Omega.Int`.
+  * [#3634](https://github.com/leanprover/lean4/pull/3634) upstreams `Std.Data.Nat`
+    and [#3635](https://github.com/leanprover/lean4/pull/3635) upstreams `Std.Data.Int`.
+  * [#3790](https://github.com/leanprover/lean4/pull/3790) reduces more imports for `omega`.
+  * [#3694](https://github.com/leanprover/lean4/pull/3694) extends `GetElem` interface with `getElem!` and `getElem?` to simplify containers like `RBMap`.
+  * [#3865](https://github.com/leanprover/lean4/pull/3865) renames `Option.toMonad` (see breaking changes below).
+  * [#3882](https://github.com/leanprover/lean4/pull/3882) unifies `lexOrd` with `compareLex`.
+* **Other fixes or improvements**
+  * [#3765](https://github.com/leanprover/lean4/pull/3765) makes `Quotient.sound` be a `theorem`.
+  * [#3645](https://github.com/leanprover/lean4/pull/3645) fixes `System.FilePath.parent` in the case of absolute paths.
+  * [#3660](https://github.com/leanprover/lean4/pull/3660) `ByteArray.toUInt64LE!` and `ByteArray.toUInt64BE!` were swapped.
+  * [#3881](https://github.com/leanprover/lean4/pull/3881), [#3887](https://github.com/leanprover/lean4/pull/3887) fix linearity issues in `HashMap.insertIfNew`, `HashSet.erase`, and `HashMap.erase`.
+    The `HashMap.insertIfNew` fix improves `import` performance.
+  * [#3830](https://github.com/leanprover/lean4/pull/3830) ensures linearity in `Parsec.many*Core`.
+  * [#3930](https://github.com/leanprover/lean4/pull/3930) adds `FS.Stream.isTty` field.
+  * [#3866](https://github.com/leanprover/lean4/pull/3866) deprecates `Option.toBool` in favor of `Option.isSome`.
+  * [#3975](https://github.com/leanprover/lean4/pull/3975) upstreams `Data.List.Init` and `Data.Array.Init` material from Std.
+  * [#3942](https://github.com/leanprover/lean4/pull/3942) adds instances that make `ac_rfl` work without Mathlib.
+  * [#4010](https://github.com/leanprover/lean4/pull/4010) changes `Fin.induction` to use structural induction.
+  * [02753f](https://github.com/leanprover/lean4/commit/02753f6e4c510c385efcbf71fa9a6bec50fce9ab)
+    fixes bug in `reduceLeDiff` simproc.
+  * [#4097](https://github.com/leanprover/lean4/pull/4097)
+    adds `IO.TaskState` and `IO.getTaskState` to get the task from the Lean runtime's task manager.
+* **Docs:** [#3615](https://github.com/leanprover/lean4/pull/3615), [#3664](https://github.com/leanprover/lean4/pull/3664),
+  [#3707](https://github.com/leanprover/lean4/pull/3707), [#3734](https://github.com/leanprover/lean4/pull/3734),
+  [#3868](https://github.com/leanprover/lean4/pull/3868), [#3861](https://github.com/leanprover/lean4/pull/3861),
+  [#3869](https://github.com/leanprover/lean4/pull/3869), [#3858](https://github.com/leanprover/lean4/pull/3858),
+  [#3856](https://github.com/leanprover/lean4/pull/3856), [#3857](https://github.com/leanprover/lean4/pull/3857),
+  [#3867](https://github.com/leanprover/lean4/pull/3867), [#3864](https://github.com/leanprover/lean4/pull/3864),
+  [#3860](https://github.com/leanprover/lean4/pull/3860), [#3859](https://github.com/leanprover/lean4/pull/3859),
+  [#3871](https://github.com/leanprover/lean4/pull/3871), [#3919](https://github.com/leanprover/lean4/pull/3919).
+
+### Lean internals
+
+* **Defeq and WHNF algorithms**
+  * [#3616](https://github.com/leanprover/lean4/pull/3616) gives better support for reducing `Nat.rec` expressions.
+  * [#3774](https://github.com/leanprover/lean4/pull/3774) add tracing for "non-easy" WHNF cases.
+  * [#3807](https://github.com/leanprover/lean4/pull/3807) fixes an `isDefEq` performance issue, now trying structure eta *after* lazy delta reduction.
+  * [#3816](https://github.com/leanprover/lean4/pull/3816) fixes `.yesWithDeltaI` behavior to prevent increasing transparency level when reducing projections.
+  * [#3837](https://github.com/leanprover/lean4/pull/3837) improves heuristic at `isDefEq`.
+  * [#3965](https://github.com/leanprover/lean4/pull/3965) improves `isDefEq` for constraints of the form `t.i =?= s.i`.
+  * [#3977](https://github.com/leanprover/lean4/pull/3977) improves `isDefEqProj`.
+  * [#3981](https://github.com/leanprover/lean4/pull/3981) adds universe constraint approximations to be able to solve `u =?= max u ?v` using `?v = u`.
+    These approximations are only applied when universe constraints cannot be postponed anymore.
+  * [#4004](https://github.com/leanprover/lean4/pull/4004) improves `isDefEqProj` during typeclass resolution.
+  * [#4012](https://github.com/leanprover/lean4/pull/4012) adds `backward.isDefEq.lazyProjDelta` and `backward.isDefEq.lazyWhnfCore` backwards compatibility flags.
+* **Kernel**
+  * [#3966](https://github.com/leanprover/lean4/pull/3966) removes dead code.
+  * [#4035](https://github.com/leanprover/lean4/pull/4035) fixes mismatch for `TheoremVal` between Lean and C++.
+* **Discrimination trees**
+  * [423fed](https://github.com/leanprover/lean4/commit/423fed79a9de75705f34b3e8648db7e076c688d7)
+    and [3218b2](https://github.com/leanprover/lean4/commit/3218b25974d33e92807af3ce42198911c256ff1d):
+    simplify handling of dependent/non-dependent pi types.
+* **Typeclass instance synthesis**
+  * [#3638](https://github.com/leanprover/lean4/pull/3638) eta-reduces synthesized instances
+  * [ce350f](https://github.com/leanprover/lean4/commit/ce350f348161e63fccde6c4a5fe1fd2070e7ce0f) fixes a linearity issue
+  * [917a31](https://github.com/leanprover/lean4/commit/917a31f694f0db44d6907cc2b1485459afe74d49)
+    improves performance by considering at most one answer for subgoals not containing metavariables.
+    [#4008](https://github.com/leanprover/lean4/pull/4008) adds `backward.synthInstance.canonInstances` backward compatibility flag.
+* **Definition processing**
+  * [#3661](https://github.com/leanprover/lean4/pull/3661), [#3767](https://github.com/leanprover/lean4/pull/3767) changes automatically generated equational theorems to be named
+    using suffix `.eq_<idx>` instead of `._eq_<idx>`, and `.eq_def` instead of `._unfold`. (See breaking changes below.)
+    [#3675](https://github.com/leanprover/lean4/pull/3675) adds a mechanism to reserve names.
+    [#3803](https://github.com/leanprover/lean4/pull/3803) fixes reserved name resolution inside namespaces and fixes handling of `match`er declarations and equation lemmas.
+  * [#3662](https://github.com/leanprover/lean4/pull/3662) causes auxiliary definitions nested inside theorems to become `def`s if they are not proofs.
+  * [#4006](https://github.com/leanprover/lean4/pull/4006) makes proposition fields of `structure`s be theorems.
+  * [#4018](https://github.com/leanprover/lean4/pull/4018) makes it an error for a theorem to be `extern`.
+  * [#4047](https://github.com/leanprover/lean4/pull/4047) improves performance making equations for well-founded recursive definitions.
+* **Refactors**
+  * [#3614](https://github.com/leanprover/lean4/pull/3614) avoids unfolding in `Lean.Meta.evalNat`.
+  * [#3621](https://github.com/leanprover/lean4/pull/3621) centralizes functionality for `Fix`/`GuessLex`/`FunInd` in the `ArgsPacker` module.
+  * [#3186](https://github.com/leanprover/lean4/pull/3186) rewrites the UnusedVariable linter to be more performant.
+  * [#3589](https://github.com/leanprover/lean4/pull/3589) removes coercion from `String` to `Name` (see breaking changes below).
+  * [#3237](https://github.com/leanprover/lean4/pull/3237) removes the `lines` field from `FileMap`.
+  * [#3951](https://github.com/leanprover/lean4/pull/3951) makes msg parameter to `throwTacticEx` optional.
+* **Diagnostics**
+  * [#4016](https://github.com/leanprover/lean4/pull/4016), [#4019](https://github.com/leanprover/lean4/pull/4019),
+    [#4020](https://github.com/leanprover/lean4/pull/4020), [#4030](https://github.com/leanprover/lean4/pull/4030),
+    [#4031](https://github.com/leanprover/lean4/pull/4031),
+    [c3714b](https://github.com/leanprover/lean4/commit/c3714bdc6d46845c0428735b283c5b48b23cbcf7),
+    [#4049](https://github.com/leanprover/lean4/pull/4049) adds `set_option diagnostics true` for diagnostic counters.
+    Tracks number of unfolded declarations, instances, reducible declarations, used instances, recursor reductions,
+    `isDefEq` heuristic applications, among others.
+    This option is suggested in exceptional situations, such as at deterministic timeout and maximum recursion depth.
+  * [283587](https://github.com/leanprover/lean4/commit/283587987ab2eb3b56fbc3a19d5f33ab9e04a2ef)
+    adds diagnostic information for `simp`.
+  * [#4043](https://github.com/leanprover/lean4/pull/4043) adds diagnostic information for congruence theorems.
+  * [#4048](https://github.com/leanprover/lean4/pull/4048) display diagnostic information
+    for `set_option diagnostics true in <tactic>` and `set_option diagnostics true in <term>`.
+* **Other features**
+  * [#3800](https://github.com/leanprover/lean4/pull/3800) adds environment extension to record which definitions use structural or well-founded recursion.
+  * [#3801](https://github.com/leanprover/lean4/pull/3801) `trace.profiler` can now export to Firefox Profiler.
+  * [#3918](https://github.com/leanprover/lean4/pull/3918), [#3953](https://github.com/leanprover/lean4/pull/3953) adds `@[builtin_doc]` attribute to make docs and location of a declaration available as a builtin.
+  * [#3939](https://github.com/leanprover/lean4/pull/3939) adds the `lean --json` CLI option to print messages as JSON.
+  * [#3075](https://github.com/leanprover/lean4/pull/3075) improves `test_extern` command.
+  * [#3970](https://github.com/leanprover/lean4/pull/3970) gives monadic generalization of `FindExpr`.
+* **Docs:** [#3743](https://github.com/leanprover/lean4/pull/3743), [#3921](https://github.com/leanprover/lean4/pull/3921),
+  [#3954](https://github.com/leanprover/lean4/pull/3954).
+* **Other fixes:** [#3622](https://github.com/leanprover/lean4/pull/3622),
+  [#3726](https://github.com/leanprover/lean4/pull/3726), [#3823](https://github.com/leanprover/lean4/pull/3823),
+  [#3897](https://github.com/leanprover/lean4/pull/3897), [#3964](https://github.com/leanprover/lean4/pull/3964),
+  [#3946](https://github.com/leanprover/lean4/pull/3946), [#4007](https://github.com/leanprover/lean4/pull/4007),
+  [#4026](https://github.com/leanprover/lean4/pull/4026).
+
+### Compiler, runtime, and FFI
+
+* [#3632](https://github.com/leanprover/lean4/pull/3632) makes it possible to allocate and free thread-local runtime resources for threads not started by Lean itself.
+* [#3627](https://github.com/leanprover/lean4/pull/3627) improves error message about compacting closures.
+* [#3692](https://github.com/leanprover/lean4/pull/3692) fixes deadlock in `IO.Promise.resolve`.
+* [#3753](https://github.com/leanprover/lean4/pull/3753) catches error code from `MoveFileEx` on Windows.
+* [#4028](https://github.com/leanprover/lean4/pull/4028) fixes a double `reset` bug in `ResetReuse` transformation.
+* [6e731b](https://github.com/leanprover/lean4/commit/6e731b4370000a8e7a5cfb675a7f3d7635d21f58)
+  removes `interpreter` copy constructor to avoid potential memory safety issues.
+
+### Lake
+
+* **TOML Lake configurations**. [#3298](https://github.com/leanprover/lean4/pull/3298), [#4104](https://github.com/leanprover/lean4/pull/4104).
+
+  Lake packages can now use TOML as a alternative configuration file format instead of Lean. If the default `lakefile.lean` is missing, Lake will also look for a `lakefile.toml`. The TOML version of the configuration supports a restricted set of the Lake configuration options, only including those which can easily mapped to a TOML data structure. The TOML syntax itself fully compiles with the TOML v1.0.0 specification.
+
+  As part of the introduction of this new feature, we have been helping maintainers of some major packages within the ecosystem switch to this format. For example, the following is Aesop's new `lakefile.toml`:
+
+
+  **[leanprover-community/aesop/lakefile.toml](https://raw.githubusercontent.com/leanprover-community/aesop/de11e0ecf372976e6d627c210573146153090d2d/lakefile.toml)**
+  ```toml
+  name = "aesop"
+  defaultTargets = ["Aesop"]
+  testRunner = "test"
+  precompileModules = false
+
+  [[require]]
+  name = "batteries"
+  git = "https://github.com/leanprover-community/batteries"
+  rev = "main"
+
+  [[lean_lib]]
+  name = "Aesop"
+
+  [[lean_lib]]
+  name = "AesopTest"
+  globs = ["AesopTest.+"]
+  leanOptions = {linter.unusedVariables = false}
+
+  [[lean_exe]]
+  name = "test"
+  srcDir = "scripts"
+  ```
+
+  To assist users who wish to transition their packages between configuration file formats, there is also a new `lake translate-config` command for migrating to/from TOML.
+
+  Running `lake translate-config toml` will produce a `lakefile.toml` version of a package's `lakefile.lean`. Any configuration options unsupported by the TOML format will be discarded during translation, but the original `lakefile.lean` will remain so that you can verify the translation looks good before deleting it.
+
+* **Build progress overhaul.** [#3835](https://github.com/leanprover/lean4/pull/3835), [#4115](https://github.com/leanprover/lean4/pull/4115), [#4127](https://github.com/leanprover/lean4/pull/4127), [#4220](https://github.com/leanprover/lean4/pull/4220), [#4232](https://github.com/leanprover/lean4/pull/4232), [#4236](https://github.com/leanprover/lean4/pull/4236).
+
+  Builds are now managed by a top-level Lake build monitor, this makes the output of Lake builds more standardized and enables producing prettier and more configurable progress reports.
+
+  As part of this change, job isolation has improved. Stray I/O and other build related errors in custom targets are now properly isolated and caught as part of their job. Import errors no longer cause Lake to abort the entire build and are instead localized to the build jobs of the modules in question.
+
+  Lake also now uses ANSI escape sequences to add color and produce progress lines that update in-place; this can be toggled on and off using `--ansi` / `--no-ansi`.
+
+
+  `--wfail` and `--iofail` options have been added that causes a build to fail if any of the jobs log a warning (`--wfail`) or produce any output or log information messages (`--iofail`). Unlike some other build systems, these options do **NOT** convert these logs into errors, and Lake does not abort jobs on such a log (i.e., dependent jobs will still continue unimpeded).
+
+* `lake test`. [#3779](https://github.com/leanprover/lean4/pull/3779).
+
+  Lake now has a built-in `test` command which will run a script or executable labelled `@[test_runner]` (in Lean) or defined as the `testRunner` (in TOML) in the root package.
+
+  Lake also provides a `lake check-test` command which will exit with code `0` if the package has a properly configured test runner or error with `1` otherwise.
+
+* `lake lean`. [#3793](https://github.com/leanprover/lean4/pull/3793).
+
+  The new command `lake lean <file> [-- <args...>]` functions like `lake env lean <file> <args...>`, except that it builds the imports of `file` before running `lean`. This makes it very useful for running test or example code that imports modules that are not guaranteed to have been built beforehand.
+
+* **Miscellaneous bug fixes and improvements**
+  * [#3609](https://github.com/leanprover/lean4/pull/3609) `LEAN_GITHASH` environment variable to override the detected Git hash for Lean when computing traces, useful for testing custom builds of Lean.
+  * [#3795](https://github.com/leanprover/lean4/pull/3795) improves relative package directory path normalization in the pre-rename check.
+  * [#3957](https://github.com/leanprover/lean4/pull/3957) fixes handling of packages that appear multiple times in a dependency tree.
+  * [#3999](https://github.com/leanprover/lean4/pull/3999) makes it an error for there to be a mismatch between a package name and what it is required as. Also adds a special message for the `std`-to-`batteries` rename.
+  * [#4033](https://github.com/leanprover/lean4/pull/4033) fixes quiet mode.
+* **Docs:** [#3704](https://github.com/leanprover/lean4/pull/3704).
+
+### DevOps
+
+* [#3536](https://github.com/leanprover/lean4/pull/3536) and [#3833](https://github.com/leanprover/lean4/pull/3833)
+  add a checklist for the release process.
+* [#3600](https://github.com/leanprover/lean4/pull/3600) runs nix-ci more uniformly.
+* [#3612](https://github.com/leanprover/lean4/pull/3612) avoids argument limits when building on Windows.
+* [#3682](https://github.com/leanprover/lean4/pull/3682) builds Lean's `.o` files in parallel to rest of core.
+* [#3601](https://github.com/leanprover/lean4/pull/3601)
+  changes the way Lean is built on Windows (see breaking changes below).
+  As a result, Lake now dynamically links executables with `supportInterpreter := true` on Windows
+  to `libleanshared.dll` and `libInit_shared.dll`. Therefore, such executables will not run
+  unless those shared libraries are co-located with the executables or part of `PATH`.
+  Running the executable via `lake exe` will ensure these libraries are part of `PATH`.
+
+  In a related change, the signature of the `nativeFacets` Lake configuration options has changed
+  from a static `Array` to a function `(shouldExport : Bool) → Array`.
+  See its docstring or Lake's [README](src/lake/README.md) for further details on the changed option.
+* [#3690](https://github.com/leanprover/lean4/pull/3690) marks "Build matrix complete" as canceled if the build is canceled.
+* [#3700](https://github.com/leanprover/lean4/pull/3700), [#3702](https://github.com/leanprover/lean4/pull/3702),
+  [#3701](https://github.com/leanprover/lean4/pull/3701), [#3834](https://github.com/leanprover/lean4/pull/3834),
+  [#3923](https://github.com/leanprover/lean4/pull/3923): fixes and improvements for std and mathlib CI.
+* [#3712](https://github.com/leanprover/lean4/pull/3712) fixes `nix build .` on macOS.
+* [#3717](https://github.com/leanprover/lean4/pull/3717) replaces `shell.nix` in devShell with `flake.nix`.
+* [#3715](https://github.com/leanprover/lean4/pull/3715) and [#3790](https://github.com/leanprover/lean4/pull/3790) add test result summaries.
+* [#3971](https://github.com/leanprover/lean4/pull/3971) prevents stage0 changes via the merge queue.
+* [#3979](https://github.com/leanprover/lean4/pull/3979) adds handling for `changes-stage0` label.
+* [#3952](https://github.com/leanprover/lean4/pull/3952) adds a script to summarize GitHub issues.
+* [18a699](https://github.com/leanprover/lean4/commit/18a69914da53dbe37c91bc2b9ce65e1dc01752b6)
+  fixes asan linking
+
+### Breaking changes
+
+* Due to the major Lake build refactor, code using the affected parts of the Lake API or relying on the previous output format of Lake builds is likely to have been broken. We have tried to minimize the breakages and, where possible, old definitions have been marked `@[deprecated]` with a reference to the new alternative.
+
+* Executables configured with `supportInterpreter := true` on Windows should now be run via `lake exe` to function properly.
+
+* Automatically generated equational theorems are now named using suffix `.eq_<idx>` instead of `._eq_<idx>`, and `.eq_def` instead of `._unfold`. Example:
+```
+def fact : Nat → Nat
+  | 0 => 1
+  | n+1 => (n+1) * fact n
+
+theorem ex : fact 0 = 1 := by unfold fact; decide
+
+#check fact.eq_1
+-- fact.eq_1 : fact 0 = 1
+
+#check fact.eq_2
+-- fact.eq_2 (n : Nat) : fact (Nat.succ n) = (n + 1) * fact n
+
+#check fact.eq_def
+/-
+fact.eq_def :
+  ∀ (x : Nat),
+    fact x =
+      match x with
+      | 0 => 1
+      | Nat.succ n => (n + 1) * fact n
+-/
+```
+
+* The coercion from `String` to `Name` was removed. Previously, it was `Name.mkSimple`, which does not separate strings at dots, but experience showed that this is not always the desired coercion. For the previous behavior, manually insert a call to `Name.mkSimple`.
+
+* The `Subarray` fields `as`, `h₁` and `h₂` have been renamed to `array`, `start_le_stop`, and `stop_le_array_size`, respectively. This more closely follows standard Lean conventions. Deprecated aliases for the field projections were added; these will be removed in a future release.
+
+* The change to the instance name algorithm (described above) can break projects that made use of the auto-generated names.
+
+* `Option.toMonad` has been renamed to `Option.getM` and the unneeded `[Monad m]` instance argument has been removed.
 
 v4.7.0
 ---------

doc/char.md

@@ -1 +1,11 @@
 # Characters
+
+A value of type `Char`, also known as a character, is a [Unicode scalar value](https://www.unicode.org/glossary/#unicode_scalar_value). It is represented using an unsigned 32-bit integer and is statically guaranteed to be a valid Unicode scalar value.
+
+Syntactically, character literals are enclosed in single quotes.
+```lean
+#eval 'a' -- 'a'
+#eval '∀' -- '∀'
+```
+
+Characters are ordered and can be decidably compared using the relational operators `=`, `<`, `≤`, `>`, `≥`.

doc/dev/release_checklist.md

@@ -46,7 +46,6 @@ We'll use `v4.6.0` as the intended release version as a running example.
   - We do this for the repositories:
     - [lean4checker](https://github.com/leanprover/lean4checker)
       - No dependencies
-      - Note: `lean4checker` uses a different version tagging scheme: use `toolchain/v4.6.0` rather than `v4.6.0`.
       - Toolchain bump PR
       - Create and push the tag
       - Merge the tag into `stable`
@@ -82,10 +81,8 @@ We'll use `v4.6.0` as the intended release version as a running example.
       - Dependencies: `Aesop`, `ProofWidgets4`, `lean4checker`, `Batteries`, `doc-gen4`, `import-graph`
       - Toolchain bump PR notes:
         - In addition to updating the `lean-toolchain` and `lakefile.lean`,
-          in `.github/workflows/build.yml.in` in the `lean4checker` section update the line
-          `git checkout toolchain/v4.6.0` to the appropriate tag,
-          and then run `.github/workflows/mk_build_yml.sh`. Coordinate with
-          a Mathlib maintainer to get this merged.
+          in `.github/workflows/lean4checker.yml` update the line
+          `git checkout v4.6.0` to the appropriate tag. 
         - Push the PR branch to the main Mathlib repository rather than a fork, or CI may not work reliably
         - Create and push the tag
         - Create a new branch from the tag, push it, and open a pull request against `stable`.

doc/examples/compiler/.gitignore

@@ -0,0 +1 @@
+build

flake.nix

@@ -42,6 +42,7 @@
 	    lean-packages.llvmPackages.llvm  # llvm-symbolizer for asan/lsan
 	    # TODO: only add when proven to not affect the flakification
 	    #pkgs.python3
+      tree  # for CI
 	  ];
 	  # https://github.com/NixOS/nixpkgs/issues/60919
 	  hardeningDisable = [ "all" ];

nix/bootstrap.nix

@@ -170,7 +170,7 @@ rec {
           ln -sf ${lean-all}/* .
         '';
         buildPhase = ''
-          ctest --output-junit test-results.xml --output-on-failure -E 'leancomptest_(doc_example|foreign)|leanlaketest_init' -j$NIX_BUILD_CORES
+          ctest --output-junit test-results.xml --output-on-failure -E 'leancomptest_(doc_example|foreign)' -j$NIX_BUILD_CORES
         '';
         installPhase = ''
           mkdir $out
@@ -178,7 +178,7 @@ rec {
         '';
       };
       update-stage0 =
-        let cTree = symlinkJoin { name = "cs"; paths = [ Init.cTree Lean.cTree ]; }; in
+        let cTree = symlinkJoin { name = "cs"; paths = [ Init.cTree Lean.cTree Lake.cTree ]; }; in
         writeShellScriptBin "update-stage0" ''
           CSRCS=${cTree} CP_C_PARAMS="--dereference --no-preserve=all" ${src + "/script/lib/update-stage0"}
         '';

releases_drafts/messagedata.md

@@ -1,13 +0,0 @@
-* The `MessageData.ofPPFormat` constructor has been removed.
-  Its functionality has been split into two:
-
-  - for lazy structured messages, please use `MessageData.lazy`;
-  - for embedding `Format` or `FormatWithInfos`, use `MessageData.ofFormatWithInfos`.
-
-  An example migration can be found in [#3929](https://github.com/leanprover/lean4/pull/3929/files#diff-5910592ab7452a0e1b2616c62d22202d2291a9ebb463145f198685aed6299867L109).
-
-* The `MessageData.ofFormat` constructor has been turned into a function.
-  If you need to inspect `MessageData`,
-  you can pattern-match on `MessageData.ofFormatWithInfos`.
-
-part of #3929

releases_drafts/varCtorNameLint.md

@@ -0,0 +1,45 @@
+A new linter flags situations where a local variable's name is one of
+the argumentless constructors of its type. This can arise when a user either
+doesn't open a namespace or doesn't add a dot or leading qualifier, as
+in the following:
+
+````
+inductive Tree (α : Type) where
+  | leaf
+  | branch (left : Tree α) (val : α) (right : Tree α)
+
+def depth : Tree α → Nat
+  | leaf => 0
+````
+
+With this linter, the `leaf` pattern is highlighted as a local
+variable whose name overlaps with the constructor `Tree.leaf`.
+
+The linter can be disabled with `set_option linter.constructorNameAsVariable false`.
+
+Additionally, the error message that occurs when a name in a pattern that takes arguments isn't valid now suggests similar names that would be valid. This means that the following definition:
+
+```
+def length (list : List α) : Nat :=
+  match list with
+  | nil => 0
+  | cons x xs => length xs + 1
+```
+
+now results in the following warning:
+
+```
+warning: Local variable 'nil' resembles constructor 'List.nil' - write '.nil' (with a dot) or 'List.nil' to use the constructor.
+note: this linter can be disabled with `set_option linter.constructorNameAsVariable false`
+```
+
+and error:
+
+```
+invalid pattern, constructor or constant marked with '[match_pattern]' expected
+
+Suggestion: 'List.cons' is similar
+```
+
+
+#4301

releases_drafts/wf.md

@@ -1,12 +0,0 @@
-Functions defined by well-founded recursion are now marked as
-`@[irreducible]`, which should prevent expensive and often unfruitful
-unfolding of such definitions.
-
-Existing proofs that hold by definitional equality (e.g. `rfl`) can be
-rewritten to explictly unfold the function definition (using `simp`,
-`unfold`, `rw`), or the recursive function can be temporariliy made
-semireducible (using `unseal f in` before the command) or the function
-definition itself can be marked as `@[semireducible]` to get the previous
-behavor.
-
-#4061

script/lib/update-stage0

@@ -15,4 +15,19 @@ for f in $(git ls-files src ':!:src/lake/*' ':!:src/Leanc.lean'); do
         cp $f stage0/$f
     fi
 done
+
+# special handling for Lake files due to its nested directory
+# copy the README to ensure the `stage0/src/lake` directory is comitted
+for f in $(git ls-files 'src/lake/Lake/*' src/lake/Lake.lean src/lake/README.md ':!:src/lakefile.toml'); do
+    if [[ $f == *.lean ]]; then
+        f=${f#src/lake}
+        f=${f%.lean}.c
+        mkdir -p $(dirname stage0/stdlib/$f)
+        cp ${CP_C_PARAMS:-} $CSRCS/$f stage0/stdlib/$f
+    else
+        mkdir -p $(dirname stage0/$f)
+        cp $f stage0/$f
+    fi
+done
+
 git add stage0

src/CMakeLists.txt

@@ -9,7 +9,7 @@ endif()
 include(ExternalProject)
 project(LEAN CXX C)
 set(LEAN_VERSION_MAJOR 4)
-set(LEAN_VERSION_MINOR 9)
+set(LEAN_VERSION_MINOR 10)
 set(LEAN_VERSION_PATCH 0)
 set(LEAN_VERSION_IS_RELEASE 0)  # This number is 1 in the release revision, and 0 otherwise.
 set(LEAN_SPECIAL_VERSION_DESC "" CACHE STRING "Additional version description like 'nightly-2018-03-11'")
@@ -73,6 +73,7 @@ option(USE_GMP "USE_GMP" ON)
 
 # development-specific options
 option(CHECK_OLEAN_VERSION "Only load .olean files compiled with the current version of Lean" OFF)
+option(USE_LAKE "Use Lake instead of lean.mk for building core libs from language server" OFF)
 
 set(LEAN_EXTRA_MAKE_OPTS  ""                           CACHE STRING "extra options to lean --make")
 set(LEANC_CC              ${CMAKE_C_COMPILER}          CACHE STRING "C compiler to use in `leanc`")
@@ -577,11 +578,7 @@ else()
   string(APPEND CMAKE_EXE_LINKER_FLAGS " -lInit_shared -lleanshared")
 endif()
 
-if(${STAGE} GREATER 0 AND NOT ${CMAKE_SYSTEM_NAME} MATCHES "Emscripten")
-  if(NOT EXISTS ${LEAN_SOURCE_DIR}/lake/Lake.lean)
-    message(FATAL_ERROR "src/lake does not exist. Please check out the Lake submodule using `git submodule update --init src/lake`.")
-  endif()
-
+if(NOT ${CMAKE_SYSTEM_NAME} MATCHES "Emscripten")
   add_custom_target(lake ALL
     WORKING_DIRECTORY ${LEAN_SOURCE_DIR}
     DEPENDS leanshared
@@ -658,3 +655,9 @@ endif()
 string(REPLACE "$" "$$" CMAKE_EXE_LINKER_FLAGS_MAKE "${CMAKE_EXE_LINKER_FLAGS}")
 string(REPLACE "$" "$$" CMAKE_EXE_LINKER_FLAGS_MAKE_MAKE "${CMAKE_EXE_LINKER_FLAGS_MAKE}")
 configure_file(${LEAN_SOURCE_DIR}/stdlib.make.in ${CMAKE_BINARY_DIR}/stdlib.make)
+
+if(USE_LAKE AND STAGE EQUAL 1)
+  configure_file(${LEAN_SOURCE_DIR}/lakefile.toml.in ${LEAN_SOURCE_DIR}/lakefile.toml)
+  configure_file(${LEAN_SOURCE_DIR}/lakefile.toml.in ${LEAN_SOURCE_DIR}/../tests/lakefile.toml)
+  configure_file(${LEAN_SOURCE_DIR}/lakefile.toml.in ${LEAN_SOURCE_DIR}/../lakefile.toml)
+endif()

src/Init/Core.lean

@@ -468,11 +468,11 @@ class Singleton (α : outParam <| Type u) (β : Type v) where
 export Singleton (singleton)
 
 /-- `insert x ∅ = {x}` -/
-class IsLawfulSingleton (α : Type u) (β : Type v) [EmptyCollection β] [Insert α β] [Singleton α β] :
+class LawfulSingleton (α : Type u) (β : Type v) [EmptyCollection β] [Insert α β] [Singleton α β] :
     Prop where
   /-- `insert x ∅ = {x}` -/
   insert_emptyc_eq (x : α) : (insert x ∅ : β) = singleton x
-export IsLawfulSingleton (insert_emptyc_eq)
+export LawfulSingleton (insert_emptyc_eq)
 
 /-- Type class used to implement the notation `{ a ∈ c | p a }` -/
 class Sep (α : outParam <| Type u) (γ : Type v) where

src/Init/Data/AC.lean

@@ -146,8 +146,8 @@ theorem Context.evalList_mergeIdem (ctx : Context α) (h : ContextInformation.is
     | nil =>
       simp [mergeIdem, mergeIdem.loop]
       split
-      case inl h₂ => simp [evalList, h₂, h.1, EvalInformation.evalOp]
-      rfl
+      next h₂ => simp [evalList, h₂, h.1, EvalInformation.evalOp]
+      next => rfl
     | cons z zs =>
       by_cases h₂ : x = y
       case pos =>
@@ -191,11 +191,11 @@ theorem Context.evalList_insert
     . simp [evalList, h.1, EvalInformation.evalOp]
   | step y z zs ih =>
     simp [insert] at *; split
-    case inl => rfl
-    case inr =>
+    next => rfl
+    next =>
       split
-      case inl => simp [evalList, EvalInformation.evalOp]; rw [h.1, ctx.assoc.1, h.1 (evalList _ _ _)]
-      case inr => simp_all [evalList, EvalInformation.evalOp]; rw [h.1, ctx.assoc.1, h.1 (evalList _ _ _)]
+      next => simp [evalList, EvalInformation.evalOp]; rw [h.1, ctx.assoc.1, h.1 (evalList _ _ _)]
+      next => simp_all [evalList, EvalInformation.evalOp]; rw [h.1, ctx.assoc.1, h.1 (evalList _ _ _)]
 
 theorem Context.evalList_sort_congr
   (ctx : Context α)

src/Init/Data/Array/Basic.lean

@@ -481,7 +481,7 @@ def all (as : Array α) (p : α → Bool) (start := 0) (stop := as.size) : Bool
   Id.run <| as.allM p start stop
 
 def contains [BEq α] (as : Array α) (a : α) : Bool :=
-  as.any fun b => a == b
+  as.any (· == a)
 
 def elem [BEq α] (a : α) (as : Array α) : Bool :=
   as.contains a
@@ -791,11 +791,11 @@ def toArrayLit (a : Array α) (n : Nat) (hsz : a.size = n) : Array α :=
 theorem ext' {as bs : Array α} (h : as.data = bs.data) : as = bs := by
   cases as; cases bs; simp at h; rw [h]
 
-theorem toArrayAux_eq (as : List α) (acc : Array α) : (as.toArrayAux acc).data = acc.data ++ as := by
+@[simp] theorem toArrayAux_eq (as : List α) (acc : Array α) : (as.toArrayAux acc).data = acc.data ++ as := by
   induction as generalizing acc <;> simp [*, List.toArrayAux, Array.push, List.append_assoc, List.concat_eq_append]
 
 theorem data_toArray (as : List α) : as.toArray.data = as := by
-  simp [List.toArray, toArrayAux_eq, Array.mkEmpty]
+  simp [List.toArray, Array.mkEmpty]
 
 theorem toArrayLit_eq (as : Array α) (n : Nat) (hsz : as.size = n) : as = toArrayLit as n hsz := by
   apply ext'

src/Init/Data/Array/BasicAux.lean

@@ -9,7 +9,7 @@ import Init.Data.Nat.Linear
 import Init.NotationExtra
 
 theorem Array.of_push_eq_push {as bs : Array α} (h : as.push a = bs.push b) : as = bs ∧ a = b := by
-  simp [push] at h
+  simp only [push, mk.injEq] at h
   have ⟨h₁, h₂⟩ := List.of_concat_eq_concat h
   cases as; cases bs
   simp_all

src/Init/Data/Array/DecidableEq.lean

@@ -27,17 +27,17 @@ decreasing_by decreasing_trivial_pre_omega
 theorem eq_of_isEqv [DecidableEq α] (a b : Array α) : Array.isEqv a b (fun x y => x = y) → a = b := by
   simp [Array.isEqv]
   split
-  case inr => intro; contradiction
-  case inl hsz =>
+  next hsz =>
    intro h
    have aux := eq_of_isEqvAux a b hsz 0 (Nat.zero_le ..) h
    exact ext a b hsz fun i h _ => aux i (Nat.zero_le ..) _
+  next => intro; contradiction
 
 theorem isEqvAux_self [DecidableEq α] (a : Array α) (i : Nat) : Array.isEqvAux a a rfl (fun x y => x = y) i = true := by
   unfold Array.isEqvAux
   split
-  case inl h => simp [h, isEqvAux_self a (i+1)]
-  case inr h => simp [h]
+  next h => simp [h, isEqvAux_self a (i+1)]
+  next h => simp [h]
 termination_by a.size - i
 decreasing_by decreasing_trivial_pre_omega
 

src/Init/Data/Array/Lemmas.lean

@@ -14,7 +14,7 @@ import Init.TacticsExtra
 /-!
 ## Bootstrapping theorems about arrays
 
-This file contains some theorems about `Array` and `List` needed for `Std.List.Basic`.
+This file contains some theorems about `Array` and `List` needed for `Init.Data.List.Impl`.
 -/
 
 namespace Array
@@ -34,9 +34,13 @@ attribute [simp] data_toArray uset
 
 @[simp] theorem size_mk (as : List α) : (Array.mk as).size = as.length := by simp [size]
 
-theorem getElem_eq_data_get (a : Array α) (h : i < a.size) : a[i] = a.data.get ⟨i, h⟩ := by
+theorem getElem_eq_data_getElem (a : Array α) (h : i < a.size) : a[i] = a.data[i] := by
   by_cases i < a.size <;> (try simp [*]) <;> rfl
 
+@[deprecated getElem_eq_data_getElem (since := "2024-06-12")]
+theorem getElem_eq_data_get (a : Array α) (h : i < a.size) : a[i] = a.data.get ⟨i, h⟩ := by
+  simp [getElem_eq_data_getElem]
+
 theorem foldlM_eq_foldlM_data.aux [Monad m]
     (f : β → α → m β) (arr : Array α) (i j) (H : arr.size ≤ i + j) (b) :
     foldlM.loop f arr arr.size (Nat.le_refl _) i j b = (arr.data.drop j).foldlM f b := by
@@ -114,11 +118,11 @@ theorem foldr_push (f : α → β → β) (init : β) (arr : Array α) (a : α)
 theorem get_push_lt (a : Array α) (x : α) (i : Nat) (h : i < a.size) :
     have : i < (a.push x).size := by simp [*, Nat.lt_succ_of_le, Nat.le_of_lt]
     (a.push x)[i] = a[i] := by
-  simp only [push, getElem_eq_data_get, List.concat_eq_append, List.get_append_left, h]
+  simp only [push, getElem_eq_data_getElem, List.concat_eq_append, List.getElem_append_left, h]
 
 @[simp] theorem get_push_eq (a : Array α) (x : α) : (a.push x)[a.size] = x := by
-  simp only [push, getElem_eq_data_get, List.concat_eq_append]
-  rw [List.get_append_right] <;> simp [getElem_eq_data_get, Nat.zero_lt_one]
+  simp only [push, getElem_eq_data_getElem, List.concat_eq_append]
+  rw [List.getElem_append_right] <;> simp [getElem_eq_data_getElem, Nat.zero_lt_one]
 
 theorem get_push (a : Array α) (x : α) (i : Nat) (h : i < (a.push x).size) :
     (a.push x)[i] = if h : i < a.size then a[i] else x := by
@@ -135,7 +139,8 @@ where
       mapM.map f arr i r = (arr.data.drop i).foldlM (fun bs a => bs.push <$> f a) r := by
     unfold mapM.map; split
     · rw [← List.get_drop_eq_drop _ i ‹_›]
-      simp [aux (i+1), map_eq_pure_bind]; rfl
+      simp only [aux (i + 1), map_eq_pure_bind, data_length, List.foldlM_cons, bind_assoc, pure_bind]
+      rfl
     · rw [List.drop_length_le (Nat.ge_of_not_lt ‹_›)]; rfl
   termination_by arr.size - i
   decreasing_by decreasing_trivial_pre_omega
@@ -233,11 +238,11 @@ theorem get!_eq_getD [Inhabited α] (a : Array α) : a.get! n = a.getD n default
 @[simp] theorem getElem_set_eq (a : Array α) (i : Fin a.size) (v : α) {j : Nat}
       (eq : i.val = j) (p : j < (a.set i v).size) :
     (a.set i v)[j]'p = v := by
-  simp [set, getElem_eq_data_get, ←eq]
+  simp [set, getElem_eq_data_getElem, ←eq]
 
 @[simp] theorem getElem_set_ne (a : Array α) (i : Fin a.size) (v : α) {j : Nat} (pj : j < (a.set i v).size)
     (h : i.val ≠ j) : (a.set i v)[j]'pj = a[j]'(size_set a i v ▸ pj) := by
-  simp only [set, getElem_eq_data_get, List.get_set_ne _ h]
+  simp only [set, getElem_eq_data_getElem, List.getElem_set_ne h]
 
 theorem getElem_set (a : Array α) (i : Fin a.size) (v : α) (j : Nat)
     (h : j < (a.set i v).size) :
@@ -321,7 +326,7 @@ termination_by n - i
 @[simp] theorem mkArray_data (n : Nat) (v : α) : (mkArray n v).data = List.replicate n v := rfl
 
 @[simp] theorem getElem_mkArray (n : Nat) (v : α) (h : i < (mkArray n v).size) :
-    (mkArray n v)[i] = v := by simp [Array.getElem_eq_data_get]
+    (mkArray n v)[i] = v := by simp [Array.getElem_eq_data_getElem]
 
 /-- # mem -/
 
@@ -332,7 +337,7 @@ theorem not_mem_nil (a : α) : ¬ a ∈ #[] := nofun
 /-- # get lemmas -/
 
 theorem getElem?_mem {l : Array α} {i : Fin l.size} : l[i] ∈ l := by
-  erw [Array.mem_def, getElem_eq_data_get]
+  erw [Array.mem_def, getElem_eq_data_getElem]
   apply List.get_mem
 
 theorem getElem_fin_eq_data_get (a : Array α) (i : Fin _) : a[i] = a.data.get i := rfl
@@ -347,7 +352,7 @@ theorem get?_len_le (a : Array α) (i : Nat) (h : a.size ≤ i) : a[i]? = none :
   simp [getElem?_neg, h]
 
 theorem getElem_mem_data (a : Array α) (h : i < a.size) : a[i] ∈ a.data := by
-  simp only [getElem_eq_data_get, List.get_mem]
+  simp only [getElem_eq_data_getElem, List.getElem_mem]
 
 theorem getElem?_eq_data_get? (a : Array α) (i : Nat) : a[i]? = a.data.get? i := by
   by_cases i < a.size <;> simp_all [getElem?_pos, getElem?_neg, List.get?_eq_get, eq_comm]; rfl
@@ -395,7 +400,7 @@ theorem get?_push {a : Array α} : (a.push x)[i]? = if i = a.size then some x el
 
 theorem get_set_eq (a : Array α) (i : Fin a.size) (v : α) :
     (a.set i v)[i.1] = v := by
-  simp only [set, getElem_eq_data_get, List.get_set_eq]
+  simp only [set, getElem_eq_data_getElem, List.getElem_set_eq]
 
 theorem get?_set_eq (a : Array α) (i : Fin a.size) (v : α) :
     (a.set i v)[i.1]? = v := by simp [getElem?_pos, i.2]
@@ -414,7 +419,7 @@ theorem get_set (a : Array α) (i : Fin a.size) (j : Nat) (hj : j < a.size) (v :
 
 @[simp] theorem get_set_ne (a : Array α) (i : Fin a.size) {j : Nat} (v : α) (hj : j < a.size)
     (h : i.1 ≠ j) : (a.set i v)[j]'(by simp [*]) = a[j] := by
-  simp only [set, getElem_eq_data_get, List.get_set_ne _ h]
+  simp only [set, getElem_eq_data_getElem, List.getElem_set_ne h]
 
 theorem getElem_setD (a : Array α) (i : Nat) (v : α) (h : i < (setD a i v).size) :
   (setD a i v)[i] = v := by
@@ -452,7 +457,7 @@ theorem swapAt!_def (a : Array α) (i : Nat) (v : α) (h : i < a.size) :
 
 @[simp] theorem getElem_pop (a : Array α) (i : Nat) (hi : i < a.pop.size) :
     a.pop[i] = a[i]'(Nat.lt_of_lt_of_le (a.size_pop ▸ hi) (Nat.sub_le _ _)) :=
-  List.get_dropLast ..
+  List.getElem_dropLast ..
 
 theorem eq_empty_of_size_eq_zero {as : Array α} (h : as.size = 0) : as = #[] := by
   apply ext
@@ -500,27 +505,28 @@ theorem size_eq_length_data (as : Array α) : as.size = as.data.length := rfl
     simp only [mkEmpty_eq, size_push] at *
     omega
 
+set_option linter.deprecated false in
 @[simp] theorem reverse_data (a : Array α) : a.reverse.data = a.data.reverse := by
   let rec go (as : Array α) (i j hj)
       (h : i + j + 1 = a.size) (h₂ : as.size = a.size)
       (H : ∀ k, as.data.get? k = if i ≤ k ∧ k ≤ j then a.data.get? k else a.data.reverse.get? k)
       (k) : (reverse.loop as i ⟨j, hj⟩).data.get? k = a.data.reverse.get? k := by
     rw [reverse.loop]; dsimp; split <;> rename_i h₁
-    · have := reverse.termination h₁
+    · have p := reverse.termination h₁
       match j with | j+1 => ?_
-      simp at *
+      simp only [Nat.add_sub_cancel] at p ⊢
       rw [(go · (i+1) j)]
       · rwa [Nat.add_right_comm i]
       · simp [size_swap, h₂]
       · intro k
         rw [← getElem?_eq_data_get?, get?_swap]
-        simp [getElem?_eq_data_get?, getElem_eq_data_get, ← List.get?_eq_get, H, Nat.le_of_lt h₁]
+        simp only [H, getElem_eq_data_get, ← List.get?_eq_get, Nat.le_of_lt h₁, getElem?_eq_data_get?]
         split <;> rename_i h₂
-        · simp [← h₂, Nat.not_le.2 (Nat.lt_succ_self _)]
-          exact (List.get?_reverse' _ _ (Eq.trans (by simp_arith) h)).symm
+        · simp only [← h₂, Nat.not_le.2 (Nat.lt_succ_self _), Nat.le_refl, and_false]
+          exact (List.get?_reverse' (j+1) i (Eq.trans (by simp_arith) h)).symm
         split <;> rename_i h₃
-        · simp [← h₃, Nat.not_le.2 (Nat.lt_succ_self _)]
-          exact (List.get?_reverse' _ _ (Eq.trans (by simp_arith) h)).symm
+        · simp only [← h₃, Nat.not_le.2 (Nat.lt_succ_self _), Nat.le_refl, false_and]
+          exact (List.get?_reverse' i (j+1) (Eq.trans (by simp_arith) h)).symm
         simp only [Nat.succ_le, Nat.lt_iff_le_and_ne.trans (and_iff_left h₃),
           Nat.lt_succ.symm.trans (Nat.lt_iff_le_and_ne.trans (and_iff_left (Ne.symm h₂)))]
     · rw [H]; split <;> rename_i h₂
@@ -529,13 +535,17 @@ theorem size_eq_length_data (as : Array α) : as.size = as.data.length := rfl
         exact (List.get?_reverse' _ _ h).symm
       · rfl
     termination_by j - i
-  simp only [reverse]; split
+  simp only [reverse]
+  split
   · match a with | ⟨[]⟩ | ⟨[_]⟩ => rfl
   · have := Nat.sub_add_cancel (Nat.le_of_not_le ‹_›)
-    refine List.ext <| go _ _ _ _ (by simp [this]) rfl fun k => ?_
-    split; {rfl}; rename_i h
-    simp [← show k < _ + 1 ↔ _ from Nat.lt_succ (n := a.size - 1), this] at h
-    rw [List.get?_eq_none.2 ‹_›, List.get?_eq_none.2 (a.data.length_reverse ▸ ‹_›)]
+    refine List.ext_get? <| go _ _ _ _ (by simp [this]) rfl fun k => ?_
+    split
+    · rfl
+    · rename_i h
+      simp only [← show k < _ + 1 ↔ _ from Nat.lt_succ (n := a.size - 1), this, Nat.zero_le,
+        true_and, Nat.not_lt] at h
+      rw [List.get?_eq_none.2 ‹_›, List.get?_eq_none.2 (a.data.length_reverse ▸ ‹_›)]
 
 /-! ### foldl / foldr -/
 
@@ -740,7 +750,7 @@ theorem mem_of_mem_filter {a : α} {l} (h : a ∈ filter p l) : a ∈ l :=
   exact this #[]
   induction l
   · simp_all [Id.run]
-  · simp_all [Id.run]
+  · simp_all [Id.run, List.filterMap_cons]
     split <;> simp_all
 
 @[simp] theorem mem_filterMap (f : α → Option β) (l : Array α) {b : β} :
@@ -765,17 +775,17 @@ theorem size_append (as bs : Array α) : (as ++ bs).size = as.size + bs.size :=
 
 theorem get_append_left {as bs : Array α} {h : i < (as ++ bs).size} (hlt : i < as.size) :
     (as ++ bs)[i] = as[i] := by
-  simp only [getElem_eq_data_get]
+  simp only [getElem_eq_data_getElem]
   have h' : i < (as.data ++ bs.data).length := by rwa [← data_length, append_data] at h
-  conv => rhs; rw [← List.get_append_left (bs:=bs.data) (h':=h')]
+  conv => rhs; rw [← List.getElem_append_left (bs := bs.data) (h' := h')]
   apply List.get_of_eq; rw [append_data]
 
 theorem get_append_right {as bs : Array α} {h : i < (as ++ bs).size} (hle : as.size ≤ i)
     (hlt : i - as.size < bs.size := Nat.sub_lt_left_of_lt_add hle (size_append .. ▸ h)) :
     (as ++ bs)[i] = bs[i - as.size] := by
-  simp only [getElem_eq_data_get]
+  simp only [getElem_eq_data_getElem]
   have h' : i < (as.data ++ bs.data).length := by rwa [← data_length, append_data] at h
-  conv => rhs; rw [← List.get_append_right (h':=h') (h:=Nat.not_lt_of_ge hle)]
+  conv => rhs; rw [← List.getElem_append_right (h' := h') (h := Nat.not_lt_of_ge hle)]
   apply List.get_of_eq; rw [append_data]
 
 @[simp] theorem append_nil (as : Array α) : as ++ #[] = as := by
@@ -983,13 +993,13 @@ theorem all_eq_true (p : α → Bool) (as : Array α) : all as p ↔ ∀ i : Fin
   simp [all_iff_forall, Fin.isLt]
 
 theorem all_def {p : α → Bool} (as : Array α) : as.all p = as.data.all p := by
-  rw [Bool.eq_iff_iff, all_eq_true, List.all_eq_true]; simp only [List.mem_iff_get]
+  rw [Bool.eq_iff_iff, all_eq_true, List.all_eq_true]; simp only [List.mem_iff_getElem]
   constructor
-  · rintro w x ⟨r, rfl⟩
-    rw [← getElem_eq_data_get]
-    apply w
+  · rintro w x ⟨r, h, rfl⟩
+    rw [← getElem_eq_data_getElem]
+    exact w ⟨r, h⟩
   · intro w i
-    exact w as[i] ⟨i, (getElem_eq_data_get as i.2).symm⟩
+    exact w as[i] ⟨i, i.2, (getElem_eq_data_getElem as i.2).symm⟩
 
 theorem all_eq_true_iff_forall_mem {l : Array α} : l.all p ↔ ∀ x, x ∈ l → p x := by
   simp only [all_def, List.all_eq_true, mem_def]

src/Init/Data/BitVec/Basic.lean

@@ -151,12 +151,12 @@ end Int
 section Syntax
 
 /-- Notation for bit vector literals. `i#n` is a shorthand for `BitVec.ofNat n i`. -/
-scoped syntax:max term:max noWs "#" noWs term:max : term
-macro_rules | `($i#$n) => `(BitVec.ofNat $n $i)
+syntax:max num noWs "#" noWs term:max : term
+macro_rules | `($i:num#$n) => `(BitVec.ofNat $n $i)
 
 /-- Unexpander for bit vector literals. -/
 @[app_unexpander BitVec.ofNat] def unexpandBitVecOfNat : Lean.PrettyPrinter.Unexpander
-  | `($(_) $n $i) => `($i#$n)
+  | `($(_) $n $i:num) => `($i:num#$n)
   | _ => throw ()
 
 /-- Notation for bit vector literals without truncation. `i#'lt` is a shorthand for `BitVec.ofNatLt i lt`. -/
@@ -198,7 +198,7 @@ instance : Add (BitVec n) := ⟨BitVec.add⟩
 Subtraction for bit vectors. This can be interpreted as either signed or unsigned subtraction
 modulo `2^n`.
 -/
-protected def sub (x y : BitVec n) : BitVec n := .ofNat n (x.toNat + (2^n - y.toNat))
+protected def sub (x y : BitVec n) : BitVec n := .ofNat n ((2^n - y.toNat) + x.toNat)
 instance : Sub (BitVec n) := ⟨BitVec.sub⟩
 
 /--
@@ -504,7 +504,7 @@ equivalent to `a * 2^s`, modulo `2^n`.
 
 SMT-Lib name: `bvshl` except this operator uses a `Nat` shift value.
 -/
-protected def shiftLeft (a : BitVec n) (s : Nat) : BitVec n := (a.toNat <<< s)#n
+protected def shiftLeft (a : BitVec n) (s : Nat) : BitVec n := BitVec.ofNat n (a.toNat <<< s)
 instance : HShiftLeft (BitVec w) Nat (BitVec w) := ⟨.shiftLeft⟩
 
 /--

src/Init/Data/BitVec/Lemmas.lean

@@ -139,15 +139,15 @@ theorem ofBool_eq_iff_eq : ∀(b b' : Bool), BitVec.ofBool b = BitVec.ofBool b'
   getLsb (x#'lt) i = x.testBit i := by
   simp [getLsb, BitVec.ofNatLt]
 
-@[simp, bv_toNat] theorem toNat_ofNat (x w : Nat) : (x#w).toNat = x % 2^w := by
+@[simp, bv_toNat] theorem toNat_ofNat (x w : Nat) : (BitVec.ofNat w x).toNat = x % 2^w := by
   simp [BitVec.toNat, BitVec.ofNat, Fin.ofNat']
 
-@[simp] theorem toFin_ofNat (x : Nat) : toFin x#w = Fin.ofNat' x (Nat.two_pow_pos w) := rfl
+@[simp] theorem toFin_ofNat (x : Nat) : toFin (BitVec.ofNat w x) = Fin.ofNat' x (Nat.two_pow_pos w) := rfl
 
 -- Remark: we don't use `[simp]` here because simproc` subsumes it for literals.
 -- If `x` and `n` are not literals, applying this theorem eagerly may not be a good idea.
 theorem getLsb_ofNat (n : Nat) (x : Nat) (i : Nat) :
-  getLsb (x#n) i = (i < n && x.testBit i) := by
+  getLsb (BitVec.ofNat n x) i = (i < n && x.testBit i) := by
   simp [getLsb, BitVec.ofNat, Fin.val_ofNat']
 
 @[simp, deprecated toNat_ofNat (since := "2024-02-22")]
@@ -244,10 +244,10 @@ theorem toInt_eq_msb_cond (x : BitVec w) :
 theorem toInt_eq_toNat_bmod (x : BitVec n) : x.toInt = Int.bmod x.toNat (2^n) := by
   simp only [toInt_eq_toNat_cond]
   split
-  case inl g =>
+  next g =>
     rw [Int.bmod_pos] <;> simp only [←Int.ofNat_emod, toNat_mod_cancel]
     omega
-  case inr g =>
+  next g =>
     rw [Int.bmod_neg] <;> simp only [←Int.ofNat_emod, toNat_mod_cancel]
     omega
 
@@ -316,19 +316,19 @@ theorem zeroExtend'_eq {x : BitVec w} (h : w ≤ v) : x.zeroExtend' h = x.zeroEx
   let ⟨x, lt_n⟩ := x
   simp [truncate, zeroExtend]
 
-@[simp] theorem zeroExtend_zero (m n : Nat) : zeroExtend m (0#n) = 0#m := by
+@[simp] theorem zeroExtend_zero (m n : Nat) : zeroExtend m 0#n = 0#m := by
   apply eq_of_toNat_eq
   simp [toNat_zeroExtend]
 
 @[simp] theorem truncate_eq (x : BitVec n) : truncate n x = x := zeroExtend_eq x
 
-@[simp] theorem ofNat_toNat (m : Nat) (x : BitVec n) : x.toNat#m = truncate m x := by
+@[simp] theorem ofNat_toNat (m : Nat) (x : BitVec n) : BitVec.ofNat m x.toNat = truncate m x := by
   apply eq_of_toNat_eq
   simp
 
 /-- Moves one-sided left toNat equality to BitVec equality. -/
 theorem toNat_eq_nat (x : BitVec w) (y : Nat)
-  : (x.toNat = y) ↔ (y < 2^w ∧ (x = y#w)) := by
+  : (x.toNat = y) ↔ (y < 2^w ∧ (x = BitVec.ofNat w y)) := by
   apply Iff.intro
   · intro eq
     simp at eq
@@ -340,7 +340,7 @@ theorem toNat_eq_nat (x : BitVec w) (y : Nat)
 
 /-- Moves one-sided right toNat equality to BitVec equality. -/
 theorem nat_eq_toNat (x : BitVec w) (y : Nat)
-  : (y = x.toNat) ↔ (y < 2^w ∧ (x = y#w)) := by
+  : (y = x.toNat) ↔ (y < 2^w ∧ (x = BitVec.ofNat w y)) := by
   rw [@eq_comm _ _ x.toNat]
   apply toNat_eq_nat
 
@@ -416,7 +416,7 @@ protected theorem extractLsb_ofFin {n} (x : Fin (2^n)) (hi lo : Nat) :
 
 @[simp]
 protected theorem extractLsb_ofNat (x n : Nat) (hi lo : Nat) :
-  extractLsb hi lo x#n = .ofNat (hi - lo + 1) ((x % 2^n) >>> lo) := by
+  extractLsb hi lo (BitVec.ofNat n x) = .ofNat (hi - lo + 1) ((x % 2^n) >>> lo) := by
   apply eq_of_getLsb_eq
   intro ⟨i, _lt⟩
   simp [BitVec.ofNat]
@@ -642,8 +642,8 @@ theorem shiftLeftZeroExtend_eq {x : BitVec w} :
     (shiftLeftZeroExtend x i).msb = x.msb := by
   simp [shiftLeftZeroExtend_eq, BitVec.msb]
 
-theorem shiftLeft_shiftLeft {w : Nat} (x : BitVec w) (n m : Nat) :
-    (x <<< n) <<< m = x <<< (n + m) := by
+theorem shiftLeft_add {w : Nat} (x : BitVec w) (n m : Nat) :
+    x <<< (n + m) = (x <<< n) <<< m := by
   ext i
   simp only [getLsb_shiftLeft, Fin.is_lt, decide_True, Bool.true_and]
   rw [show i - (n + m) = (i - m - n) by omega]
@@ -653,6 +653,11 @@ theorem shiftLeft_shiftLeft {w : Nat} (x : BitVec w) (n m : Nat) :
   cases h₅ : decide (i < n + m) <;>
     simp at * <;> omega
 
+@[deprecated shiftLeft_add (since := "2024-06-02")]
+theorem shiftLeft_shiftLeft {w : Nat} (x : BitVec w) (n m : Nat) :
+    (x <<< n) <<< m = x <<< (n + m) := by
+  rw [shiftLeft_add]
+
 /-! ### ushiftRight -/
 
 @[simp, bv_toNat] theorem toNat_ushiftRight (x : BitVec n) (i : Nat) :
@@ -726,6 +731,59 @@ theorem getLsb_sshiftRight (x : BitVec w) (s i : Nat) :
         Nat.not_lt, decide_eq_true_eq]
       omega
 
+/-! ### signExtend -/
+
+/-- Equation theorem for `Int.sub` when both arguments are `Int.ofNat` -/
+private theorem Int.ofNat_sub_ofNat_of_lt {n m : Nat} (hlt : n < m) :
+    (n : Int) - (m : Int) = -(↑(m - 1 - n) + 1) := by
+  omega
+
+/-- Equation theorem for `Int.mod` -/
+private theorem Int.negSucc_emod (m : Nat) (n : Int) :
+    -(m + 1) % n = Int.subNatNat (Int.natAbs n) ((m % Int.natAbs n) + 1) := rfl
+
+/-- The sign extension is the same as zero extending when `msb = false`. -/
+theorem signExtend_eq_not_zeroExtend_not_of_msb_false {x : BitVec w} {v : Nat} (hmsb : x.msb = false) :
+    x.signExtend v = x.zeroExtend v := by
+  ext i
+  by_cases hv : i < v
+  · simp only [signExtend, getLsb, getLsb_zeroExtend, hv, decide_True, Bool.true_and, toNat_ofInt,
+      BitVec.toInt_eq_msb_cond, hmsb, ↓reduceIte]
+    rw [Int.ofNat_mod_ofNat, Int.toNat_ofNat, Nat.testBit_mod_two_pow]
+    simp [BitVec.testBit_toNat]
+  · simp only [getLsb_zeroExtend, hv, decide_False, Bool.false_and]
+    apply getLsb_ge
+    omega
+
+/--
+The sign extension is a bitwise not, followed by a zero extend, followed by another bitwise not
+when `msb = true`. The double bitwise not ensures that the high bits are '1',
+and the lower bits are preserved. -/
+theorem signExtend_eq_not_zeroExtend_not_of_msb_true {x : BitVec w} {v : Nat} (hmsb : x.msb = true) :
+    x.signExtend v = ~~~((~~~x).zeroExtend v) := by
+  apply BitVec.eq_of_toNat_eq
+  simp only [signExtend, BitVec.toInt_eq_msb_cond, toNat_ofInt, toNat_not,
+    toNat_truncate, hmsb, ↓reduceIte]
+  norm_cast
+  rw [Int.ofNat_sub_ofNat_of_lt, Int.negSucc_emod]
+  simp only [Int.natAbs_ofNat, Nat.succ_eq_add_one]
+  rw [Int.subNatNat_of_le]
+  · rw [Int.toNat_ofNat, Nat.add_comm, Nat.sub_add_eq]
+  · apply Nat.le_trans
+    · apply Nat.succ_le_of_lt
+      apply Nat.mod_lt
+      apply Nat.two_pow_pos
+    · apply Nat.le_refl
+  · omega
+
+@[simp] theorem getLsb_signExtend (x  : BitVec w) {v i : Nat} :
+    (x.signExtend v).getLsb i = (decide (i < v) && if i < w then x.getLsb i else x.msb) := by
+  rcases hmsb : x.msb with rfl | rfl
+  · rw [signExtend_eq_not_zeroExtend_not_of_msb_false hmsb]
+    by_cases (i < v) <;> by_cases (i < w) <;> simp_all <;> omega
+  · rw [signExtend_eq_not_zeroExtend_not_of_msb_true hmsb]
+    by_cases (i < v) <;> by_cases (i < w) <;> simp_all <;> omega
+
 /-! ### append -/
 
 theorem append_def (x : BitVec v) (y : BitVec w) :
@@ -802,11 +860,16 @@ theorem msb_append {x : BitVec w} {y : BitVec v} :
   simp only [getLsb_append, cond_eq_if]
   split <;> simp [*]
 
-theorem shiftRight_shiftRight {w : Nat} (x : BitVec w) (n m : Nat) :
-    (x >>> n) >>> m = x >>> (n + m) := by
+theorem shiftRight_add {w : Nat} (x : BitVec w) (n m : Nat) :
+    x >>> (n + m) = (x >>> n) >>> m:= by
   ext i
   simp [Nat.add_assoc n m i]
 
+@[deprecated shiftRight_add (since := "2024-06-02")]
+theorem shiftRight_shiftRight {w : Nat} (x : BitVec w) (n m : Nat) :
+    (x >>> n) >>> m = x >>> (n + m) := by
+  rw [shiftRight_add]
+
 /-! ### rev -/
 
 theorem getLsb_rev (x : BitVec w) (i : Fin w) :
@@ -945,10 +1008,10 @@ Definition of bitvector addition as a nat.
 @[simp] theorem add_ofFin (x : BitVec n) (y : Fin (2^n)) :
   x + .ofFin y = .ofFin (x.toFin + y) := rfl
 
-theorem ofNat_add {n} (x y : Nat) : (x + y)#n = x#n + y#n := by
+theorem ofNat_add {n} (x y : Nat) : BitVec.ofNat n (x + y) = BitVec.ofNat n x + BitVec.ofNat n y := by
   apply eq_of_toNat_eq ; simp [BitVec.ofNat]
 
-theorem ofNat_add_ofNat {n} (x y : Nat) : x#n + y#n = (x + y)#n :=
+theorem ofNat_add_ofNat {n} (x y : Nat) : BitVec.ofNat n x + BitVec.ofNat n y = BitVec.ofNat n (x + y) :=
   (ofNat_add x y).symm
 
 protected theorem add_assoc (x y z : BitVec n) : x + y + z = x + (y + z) := by
@@ -982,10 +1045,10 @@ theorem ofInt_add {n} (x y : Int) : BitVec.ofInt n (x + y) =
 
 /-! ### sub/neg -/
 
-theorem sub_def {n} (x y : BitVec n) : x - y = .ofNat n (x.toNat + (2^n - y.toNat)) := by rfl
+theorem sub_def {n} (x y : BitVec n) : x - y = .ofNat n ((2^n - y.toNat) + x.toNat) := by rfl
 
 @[simp, bv_toNat] theorem toNat_sub {n} (x y : BitVec n) :
-  (x - y).toNat = ((x.toNat + (2^n - y.toNat)) % 2^n) := rfl
+  (x - y).toNat = (((2^n - y.toNat) + x.toNat) % 2^n) := rfl
 @[simp] theorem toFin_sub (x y : BitVec n) : (x - y).toFin = toFin x - toFin y := rfl
 
 @[simp] theorem ofFin_sub (x : Fin (2^n)) (y : BitVec n) : .ofFin x - y = .ofFin (x - y.toFin) :=
@@ -994,15 +1057,15 @@ theorem sub_def {n} (x y : BitVec n) : x - y = .ofNat n (x.toNat + (2^n - y.toNa
   rfl
 -- Remark: we don't use `[simp]` here because simproc` subsumes it for literals.
 -- If `x` and `n` are not literals, applying this theorem eagerly may not be a good idea.
-theorem ofNat_sub_ofNat {n} (x y : Nat) : x#n - y#n = .ofNat n (x + (2^n - y % 2^n)) := by
+theorem ofNat_sub_ofNat {n} (x y : Nat) : BitVec.ofNat n x - BitVec.ofNat n y = .ofNat n ((2^n - y % 2^n) + x) := by
   apply eq_of_toNat_eq ; simp [BitVec.ofNat]
 
-@[simp] protected theorem sub_zero (x : BitVec n) : x - (0#n) = x := by apply eq_of_toNat_eq ; simp
+@[simp] protected theorem sub_zero (x : BitVec n) : x - 0#n = x := by apply eq_of_toNat_eq ; simp
 
 @[simp] protected theorem sub_self (x : BitVec n) : x - x = 0#n := by
   apply eq_of_toNat_eq
   simp only [toNat_sub]
-  rw [Nat.add_sub_of_le]
+  rw [Nat.add_comm, Nat.add_sub_of_le]
   · simp
   · exact Nat.le_of_lt x.isLt
 
@@ -1016,14 +1079,15 @@ theorem ofNat_sub_ofNat {n} (x y : Nat) : x#n - y#n = .ofNat n (x + (2^n - y % 2
 theorem sub_toAdd {n} (x y : BitVec n) : x - y = x + - y := by
   apply eq_of_toNat_eq
   simp
+  rw [Nat.add_comm]
 
-@[simp] theorem neg_zero (n:Nat) : -0#n = 0#n := by apply eq_of_toNat_eq ; simp
+@[simp] theorem neg_zero (n:Nat) : -BitVec.ofNat n 0 = BitVec.ofNat n 0 := by apply eq_of_toNat_eq ; simp
 
 theorem add_sub_cancel (x y : BitVec w) : x + y - y = x := by
   apply eq_of_toNat_eq
   have y_toNat_le := Nat.le_of_lt y.isLt
-  rw [toNat_sub, toNat_add, Nat.mod_add_mod, Nat.add_assoc, ← Nat.add_sub_assoc y_toNat_le,
-    Nat.add_sub_cancel_left, Nat.add_mod_right, toNat_mod_cancel]
+  rw [toNat_sub, toNat_add, Nat.add_comm, Nat.mod_add_mod, Nat.add_assoc, ← Nat.add_sub_assoc y_toNat_le,
+      Nat.add_sub_cancel_left, Nat.add_mod_right, toNat_mod_cancel]
 
 theorem sub_add_cancel (x y : BitVec w) : x - y + y = x := by
   rw [sub_toAdd, BitVec.add_assoc, BitVec.add_comm _ y,
@@ -1094,7 +1158,7 @@ theorem ofInt_mul {n} (x y : Int) : BitVec.ofInt n (x * y) =
   x ≤ BitVec.ofFin y ↔ x.toFin ≤ y := Iff.rfl
 @[simp] theorem ofFin_le (x : Fin (2^n)) (y : BitVec n) :
   BitVec.ofFin x ≤ y ↔ x ≤ y.toFin := Iff.rfl
-@[simp] theorem ofNat_le_ofNat {n} (x y : Nat) : (x#n) ≤ (y#n) ↔ x % 2^n ≤ y % 2^n := by
+@[simp] theorem ofNat_le_ofNat {n} (x y : Nat) : (BitVec.ofNat n x) ≤ (BitVec.ofNat n y) ↔ x % 2^n ≤ y % 2^n := by
   simp [le_def]
 
 @[bv_toNat] theorem lt_def (x y : BitVec n) :
@@ -1104,7 +1168,7 @@ theorem ofInt_mul {n} (x y : Int) : BitVec.ofInt n (x * y) =
   x < BitVec.ofFin y ↔ x.toFin < y := Iff.rfl
 @[simp] theorem ofFin_lt (x : Fin (2^n)) (y : BitVec n) :
   BitVec.ofFin x < y ↔ x < y.toFin := Iff.rfl
-@[simp] theorem ofNat_lt_ofNat {n} (x y : Nat) : (x#n) < (y#n) ↔ x % 2^n < y % 2^n := by
+@[simp] theorem ofNat_lt_ofNat {n} (x y : Nat) : BitVec.ofNat n x < BitVec.ofNat n y ↔ x % 2^n < y % 2^n := by
   simp [lt_def]
 
 protected theorem lt_of_le_ne (x y : BitVec n) (h1 : x <= y) (h2 : ¬ x = y) : x < y := by
@@ -1117,7 +1181,7 @@ protected theorem lt_of_le_ne (x y : BitVec n) (h1 : x <= y) (h2 : ¬ x = y) : x
 /-! ### intMax -/
 
 /-- The bitvector of width `w` that has the largest value when interpreted as an integer. -/
-def intMax (w : Nat) : BitVec w  := (2^w - 1)#w
+def intMax (w : Nat) : BitVec w := BitVec.ofNat w (2^w - 1)
 
 theorem getLsb_intMax_eq (w : Nat) : (intMax w).getLsb i = decide (i < w) := by
   simp [intMax, getLsb]

src/Init/Data/Char/Basic.lean

@@ -40,7 +40,7 @@ theorem isValidUInt32 (n : Nat) (h : isValidCharNat n) : n < UInt32.size := by
     apply Nat.lt_trans h₂
     decide
 
-theorem isValidChar_of_isValidChar_Nat (n : Nat) (h : isValidCharNat n) : isValidChar (UInt32.ofNat' n (isValidUInt32 n h)) :=
+theorem isValidChar_of_isValidCharNat (n : Nat) (h : isValidCharNat n) : isValidChar (UInt32.ofNat' n (isValidUInt32 n h)) :=
   match h with
   | Or.inl h        => Or.inl h
   | Or.inr ⟨h₁, h₂⟩ => Or.inr ⟨h₁, h₂⟩
@@ -52,6 +52,13 @@ theorem isValidChar_zero : isValidChar 0 :=
 @[inline] def toNat (c : Char) : Nat :=
   c.val.toNat
 
+/-- Convert a character into a `UInt8`, by truncating (reducing modulo 256) if necessary. -/
+@[inline] def toUInt8 (c : Char) : UInt8 :=
+  c.val.toUInt8
+
+/-- The numbers from 0 to 256 are all valid UTF-8 characters, so we can embed one in the other. -/
+def ofUInt8 (n : UInt8) : Char := ⟨n.toUInt32, .inl (Nat.lt_trans n.1.2 (by decide))⟩
+
 instance : Inhabited Char where
   default := 'A'
 

src/Init/Data/Char/Lemmas.lean

@@ -22,4 +22,20 @@ protected theorem le_total (a b : Char) : a ≤ b ∨ b ≤ a := UInt32.le_total
 protected theorem lt_asymm {a b : Char} (h : a < b) : ¬ b < a := UInt32.lt_asymm h
 protected theorem ne_of_lt {a b : Char} (h : a < b) : a ≠ b := Char.ne_of_val_ne (UInt32.ne_of_lt h)
 
+theorem utf8Size_eq (c : Char) : c.utf8Size = 1 ∨ c.utf8Size = 2 ∨ c.utf8Size = 3 ∨ c.utf8Size = 4 := by
+  have := c.utf8Size_pos
+  have := c.utf8Size_le_four
+  omega
+
+@[simp] theorem ofNat_toNat (c : Char) : Char.ofNat c.toNat = c := by
+  rw [Char.ofNat, dif_pos]
+  rfl
+
+@[ext] theorem Char.ext : {a b : Char} → a.val = b.val → a = b
+  | ⟨_,_⟩, ⟨_,_⟩, rfl => rfl
+
+theorem Char.ext_iff {x y : Char} : x = y ↔ x.val = y.val := ⟨congrArg _, Char.ext⟩
+
 end Char
+
+@[deprecated Char.utf8Size (since := "2024-06-04")] abbrev String.csize := Char.utf8Size

src/Init/Data/Fin/Basic.lean

@@ -66,7 +66,24 @@ protected def mul : Fin n → Fin n → Fin n
 
 /-- Subtraction modulo `n` -/
 protected def sub : Fin n → Fin n → Fin n
-  | ⟨a, h⟩, ⟨b, _⟩ => ⟨(a + (n - b)) % n, mlt h⟩
+  /-
+  The definition of `Fin.sub` has been updated to improve performance.
+  The right-hand-side of the following `match` was originally
+  ```
+  ⟨(a + (n - b)) % n, mlt h⟩
+  ```
+  This caused significant performance issues when testing definitional equality,
+  such as `x =?= x - 1` where `x : Fin n` and `n` is a big number,
+  as Lean spent a long time reducing
+  ```
+  ((n - 1) + x.val) % n
+  ```
+  For example, this was an issue for `Fin 2^64` (i.e., `UInt64`).
+  This change improves performance by leveraging the fact that `Nat.add` is defined
+  using recursion on the second argument.
+  See issue #4413.
+  -/
+  | ⟨a, h⟩, ⟨b, _⟩ => ⟨((n - b) + a) % n, mlt h⟩
 
 /-!
 Remark: land/lor can be defined without using (% n), but
@@ -193,4 +210,7 @@ theorem val_add_one_le_of_lt {n : Nat} {a b : Fin n} (h : a < b) : (a : Nat) + 1
 
 theorem val_add_one_le_of_gt {n : Nat} {a b : Fin n} (h : a > b) : (b : Nat) + 1 ≤ (a : Nat) := h
 
+theorem exists_iff {p : Fin n → Prop} : (Exists fun i => p i) ↔ Exists fun i => Exists fun h => p ⟨i, h⟩ :=
+  ⟨fun ⟨⟨i, hi⟩, hpi⟩ => ⟨i, hi, hpi⟩, fun ⟨i, hi, hpi⟩ => ⟨⟨i, hi⟩, hpi⟩⟩
+
 end Fin

src/Init/Data/Fin/Lemmas.lean

@@ -24,7 +24,7 @@ theorem mod_def (a m : Fin n) : a % m = Fin.mk (a % m) (Nat.lt_of_le_of_lt (Nat.
 
 theorem mul_def (a b : Fin n) : a * b = Fin.mk ((a * b) % n) (Nat.mod_lt _ a.size_pos) := rfl
 
-theorem sub_def (a b : Fin n) : a - b = Fin.mk ((a + (n - b)) % n) (Nat.mod_lt _ a.size_pos) := rfl
+theorem sub_def (a b : Fin n) : a - b = Fin.mk (((n - b) + a) % n) (Nat.mod_lt _ a.size_pos) := rfl
 
 theorem size_pos' : ∀ [Nonempty (Fin n)], 0 < n | ⟨i⟩ => i.size_pos
 
@@ -43,9 +43,6 @@ theorem ext_iff {a b : Fin n} : a = b ↔ a.1 = b.1 := val_inj.symm
 
 theorem val_ne_iff {a b : Fin n} : a.1 ≠ b.1 ↔ a ≠ b := not_congr val_inj
 
-theorem exists_iff {p : Fin n → Prop} : (∃ i, p i) ↔ ∃ i h, p ⟨i, h⟩ :=
-  ⟨fun ⟨⟨i, hi⟩, hpi⟩ => ⟨i, hi, hpi⟩, fun ⟨i, hi, hpi⟩ => ⟨⟨i, hi⟩, hpi⟩⟩
-
 theorem forall_iff {p : Fin n → Prop} : (∀ i, p i) ↔ ∀ i h, p ⟨i, h⟩ :=
   ⟨fun h i hi => h ⟨i, hi⟩, fun h ⟨i, hi⟩ => h i hi⟩
 
@@ -381,7 +378,7 @@ theorem castSucc_lt_succ (i : Fin n) : Fin.castSucc i < i.succ :=
   lt_def.2 <| by simp only [coe_castSucc, val_succ, Nat.lt_succ_self]
 
 theorem le_castSucc_iff {i : Fin (n + 1)} {j : Fin n} : i ≤ Fin.castSucc j ↔ i < j.succ := by
-  simpa [lt_def, le_def] using Nat.succ_le_succ_iff.symm
+  simpa only [lt_def, le_def] using Nat.add_one_le_add_one_iff.symm
 
 theorem castSucc_lt_iff_succ_le {n : Nat} {i : Fin n} {j : Fin (n + 1)} :
     Fin.castSucc i < j ↔ i.succ ≤ j := .rfl
@@ -762,16 +759,16 @@ theorem addCases_right {m n : Nat} {motive : Fin (m + n) → Sort _} {left right
 
 /-! ### sub -/
 
-protected theorem coe_sub (a b : Fin n) : ((a - b : Fin n) : Nat) = (a + (n - b)) % n := by
+protected theorem coe_sub (a b : Fin n) : ((a - b : Fin n) : Nat) = ((n - b) + a) % n := by
   cases a; cases b; rfl
 
 @[simp] theorem ofNat'_sub (x : Nat) (lt : 0 < n) (y : Fin n) :
-    Fin.ofNat' x lt - y = Fin.ofNat' (x + (n - y.val)) lt := by
+    Fin.ofNat' x lt - y = Fin.ofNat' ((n - y.val) + x) lt := by
   apply Fin.eq_of_val_eq
   simp [Fin.ofNat', Fin.sub_def]
 
 @[simp] theorem sub_ofNat' (x : Fin n) (y : Nat) (lt : 0 < n) :
-    x - Fin.ofNat' y lt = Fin.ofNat' (x.val + (n - y % n)) lt := by
+    x - Fin.ofNat' y lt = Fin.ofNat' ((n - y % n) + x.val) lt := by
   apply Fin.eq_of_val_eq
   simp [Fin.ofNat', Fin.sub_def]
 
@@ -782,7 +779,7 @@ private theorem _root_.Nat.mod_eq_sub_of_lt_two_mul {x n} (h₁ : n ≤ x) (h₂
 theorem coe_sub_iff_le {a b : Fin n} : (↑(a - b) : Nat) = a - b ↔ b ≤ a := by
   rw [sub_def, le_def]
   dsimp only
-  if h : n ≤ a + (n - b) then
+  if h : n ≤ (n - b) + a then
     rw [Nat.mod_eq_sub_of_lt_two_mul h]
     all_goals omega
   else
@@ -792,7 +789,7 @@ theorem coe_sub_iff_le {a b : Fin n} : (↑(a - b) : Nat) = a - b ↔ b ≤ a :=
 theorem coe_sub_iff_lt {a b : Fin n} : (↑(a - b) : Nat) = n + a - b ↔ a < b := by
   rw [sub_def, lt_def]
   dsimp only
-  if h : n ≤ a + (n - b) then
+  if h : n ≤ (n - b) + a then
     rw [Nat.mod_eq_sub_of_lt_two_mul h]
     all_goals omega
   else

src/Init/Data/Format/Syntax.lean

@@ -20,24 +20,27 @@ private def formatInfo (showInfo : Bool) (info : SourceInfo) (f : Format) : Form
   | true, SourceInfo.synthetic pos endPos false     => f!"{pos}:{f}:{endPos}"
   | _,    _                                         => f
 
-partial def formatStxAux (maxDepth : Option Nat) (showInfo : Bool) : Nat → Syntax → Format
-  | _,     atom info val     => formatInfo showInfo info $ format (repr val)
-  | _,     ident info _ val _   => formatInfo showInfo info $ format "`" ++ format val
-  | _,     missing           => "<missing>"
-  | depth, node _ kind args  =>
+partial def formatStxAux (maxDepth : Option Nat) (showInfo : Bool) (depth : Nat) : Syntax → Format
+  | atom info val        => formatInfo showInfo info <| format (repr val)
+  | ident info _ val _   => formatInfo showInfo info <| format "`" ++ format val
+  | missing              => "<missing>"
+  | node info kind args  =>
     let depth := depth + 1;
     if kind == nullKind then
-      sbracket $
+      sbracket <|
         if args.size > 0 && depth > maxDepth.getD depth then
           ".."
         else
           joinSep (args.toList.map (formatStxAux maxDepth showInfo depth)) line
     else
-      let shorterName := kind.replacePrefix `Lean.Parser Name.anonymous;
-      let header      := format shorterName;
+      let shorterName := kind.replacePrefix `Lean.Parser Name.anonymous
+      let header      := formatInfo showInfo info <| format shorterName
       let body : List Format :=
-        if args.size > 0 && depth > maxDepth.getD depth then [".."] else args.toList.map (formatStxAux maxDepth showInfo depth);
-      paren $ joinSep (header :: body) line
+        if args.size > 0 && depth > maxDepth.getD depth then
+          [".."]
+        else
+          args.toList.map (formatStxAux maxDepth showInfo depth)
+      paren <| joinSep (header :: body) line
 
 /-- Pretty print the given syntax `stx` as a `Format`.
 Nodes deeper than `maxDepth` are omitted.

src/Init/Data/Int/DivModLemmas.lean

@@ -1075,9 +1075,9 @@ theorem emod_mul_bmod_congr (x : Int) (n : Nat) : Int.bmod (x%n * y) n = Int.bmo
 theorem bmod_add_bmod_congr : Int.bmod (Int.bmod x n + y) n = Int.bmod (x + y) n := by
   rw [bmod_def x n]
   split
-  case inl p =>
+  next p =>
     simp only [emod_add_bmod_congr]
-  case inr p =>
+  next p =>
     rw [Int.sub_eq_add_neg, Int.add_right_comm, ←Int.sub_eq_add_neg]
     simp
 
@@ -1088,9 +1088,9 @@ theorem bmod_add_bmod_congr : Int.bmod (Int.bmod x n + y) n = Int.bmod (x + y) n
 theorem bmod_mul_bmod : Int.bmod (Int.bmod x n * y) n = Int.bmod (x * y) n := by
   rw [bmod_def x n]
   split
-  case inl p =>
+  next p =>
     simp
-  case inr p =>
+  next p =>
     rw [Int.sub_mul, Int.sub_eq_add_neg, ← Int.mul_neg]
     simp
 

src/Init/Data/List.lean

@@ -10,3 +10,4 @@ import Init.Data.List.Control
 import Init.Data.List.Lemmas
 import Init.Data.List.Impl
 import Init.Data.List.TakeDrop
+import Init.Data.List.Notation

src/Init/Data/List/BasicAux.lean

@@ -5,7 +5,6 @@ Author: Leonardo de Moura
 -/
 prelude
 import Init.Data.Nat.Linear
-import Init.Ext
 
 universe u
 
@@ -13,6 +12,10 @@ namespace List
 /-! The following functions can't be defined at `Init.Data.List.Basic`, because they depend on `Init.Util`,
    and `Init.Util` depends on `Init.Data.List.Basic`. -/
 
+/-! ## Alternative getters -/
+
+/-! ### get! -/
+
 /--
 Returns the `i`-th element in the list (zero-based).
 
@@ -24,108 +27,12 @@ def get! [Inhabited α] : (as : List α) → (i : Nat) → α
   | _::as, n+1 => get! as n
   | _,     _   => panic! "invalid index"
 
-/--
-Returns the `i`-th element in the list (zero-based).
+theorem get!_nil [Inhabited α] (n : Nat) : [].get! n = (default : α) := rfl
+theorem get!_cons_succ [Inhabited α] (l : List α) (a : α) (n : Nat) :
+    (a::l).get! (n+1) = get! l n := rfl
+theorem get!_cons_zero [Inhabited α] (l : List α) (a : α) : (a::l).get! 0 = a := rfl
 
-If the index is out of bounds (`i ≥ as.length`), this function returns `none`.
-Also see `get`, `getD` and `get!`.
--/
-def get? : (as : List α) → (i : Nat) → Option α
-  | a::_,  0   => some a
-  | _::as, n+1 => get? as n
-  | _,     _   => none
-
-/--
-Returns the `i`-th element in the list (zero-based).
-
-If the index is out of bounds (`i ≥ as.length`), this function returns `fallback`.
-See also `get?` and `get!`.
--/
-def getD (as : List α) (i : Nat) (fallback : α) : α :=
-  (as.get? i).getD fallback
-
-@[ext] theorem ext : ∀ {l₁ l₂ : List α}, (∀ n, l₁.get? n = l₂.get? n) → l₁ = l₂
-  | [], [], _ => rfl
-  | a :: l₁, [], h => nomatch h 0
-  | [], a' :: l₂, h => nomatch h 0
-  | a :: l₁, a' :: l₂, h => by
-    have h0 : some a = some a' := h 0
-    injection h0 with aa; simp only [aa, ext fun n => h (n+1)]
-
-/--
-Returns the first element in the list.
-
-If the list is empty, this function panics when executed, and returns `default`.
-See `head` and `headD` for safer alternatives.
--/
-def head! [Inhabited α] : List α → α
-  | []   => panic! "empty list"
-  | a::_ => a
-
-/--
-Returns the first element in the list.
-
-If the list is empty, this function returns `none`.
-Also see `headD` and `head!`.
--/
-def head? : List α → Option α
-  | []   => none
-  | a::_ => some a
-
-/--
-Returns the first element in the list.
-
-If the list is empty, this function returns `fallback`.
-Also see `head?` and `head!`.
--/
-def headD : (as : List α) → (fallback : α) → α
-  | [],   fallback => fallback
-  | a::_, _  => a
-
-/--
-Returns the first element of a non-empty list.
--/
-def head : (as : List α) → as ≠ [] → α
-  | a::_, _ => a
-
-/--
-Drops the first element of the list.
-
-If the list is empty, this function panics when executed, and returns the empty list.
-See `tail` and `tailD` for safer alternatives.
--/
-def tail! : List α → List α
-  | []    => panic! "empty list"
-  | _::as => as
-
-/--
-Drops the first element of the list.
-
-If the list is empty, this function returns `none`.
-Also see `tailD` and `tail!`.
--/
-def tail? : List α → Option (List α)
-  | []    => none
-  | _::as => some as
-
-/--
-Drops the first element of the list.
-
-If the list is empty, this function returns `fallback`.
-Also see `head?` and `head!`.
--/
-def tailD (list fallback : List α) : List α :=
-  match list with
-  | [] => fallback
-  | _ :: tl => tl
-
-/--
-Returns the last element of a non-empty list.
--/
-def getLast : ∀ (as : List α), as ≠ [] → α
-  | [],       h => absurd rfl h
-  | [a],      _ => a
-  | _::b::as, _ => getLast (b::as) (fun h => List.noConfusion h)
+/-! ### getLast! -/
 
 /--
 Returns the last element in the list.
@@ -137,61 +44,118 @@ def getLast! [Inhabited α] : List α → α
   | []    => panic! "empty list"
   | a::as => getLast (a::as) (fun h => List.noConfusion h)
 
-/--
-Returns the last element in the list.
+/-! ## Head and tail -/
 
-If the list is empty, this function returns `none`.
-Also see `getLastD` and `getLast!`.
--/
-def getLast? : List α → Option α
-  | []    => none
-  | a::as => some (getLast (a::as) (fun h => List.noConfusion h))
+/-! ### head! -/
 
 /--
-Returns the last element in the list.
+Returns the first element in the list.
 
-If the list is empty, this function returns `fallback`.
-Also see `getLast?` and `getLast!`.
+If the list is empty, this function panics when executed, and returns `default`.
+See `head` and `headD` for safer alternatives.
 -/
-def getLastD : (as : List α) → (fallback : α) → α
-  | [],   a₀ => a₀
-  | a::as, _ => getLast (a::as) (fun h => List.noConfusion h)
+def head! [Inhabited α] : List α → α
+  | []   => panic! "empty list"
+  | a::_ => a
+
+/-! ### tail! -/
 
 /--
-`O(n)`. Rotates the elements of `xs` to the left such that the element at
-`xs[i]` rotates to `xs[(i - n) % l.length]`.
-* `rotateLeft [1, 2, 3, 4, 5] 3 = [4, 5, 1, 2, 3]`
-* `rotateLeft [1, 2, 3, 4, 5] 5 = [1, 2, 3, 4, 5]`
-* `rotateLeft [1, 2, 3, 4, 5] = [2, 3, 4, 5, 1]`
+Drops the first element of the list.
+
+If the list is empty, this function panics when executed, and returns the empty list.
+See `tail` and `tailD` for safer alternatives.
 -/
-def rotateLeft (xs : List α) (n : Nat := 1) : List α :=
-  let len := xs.length
-  if len ≤ 1 then
-    xs
-  else
-    let n := n % len
-    let b := xs.take n
-    let e := xs.drop n
-    e ++ b
+def tail! : List α → List α
+  | []    => panic! "empty list"
+  | _::as => as
+
+@[simp] theorem tail!_cons : @tail! α (a::l) = l := rfl
+
+/-! ### partitionM -/
 
 /--
-`O(n)`. Rotates the elements of `xs` to the right such that the element at
-`xs[i]` rotates to `xs[(i + n) % l.length]`.
-* `rotateRight [1, 2, 3, 4, 5] 3 = [3, 4, 5, 1, 2]`
-* `rotateRight [1, 2, 3, 4, 5] 5 = [1, 2, 3, 4, 5]`
-* `rotateRight [1, 2, 3, 4, 5] = [5, 1, 2, 3, 4]`
--/
-def rotateRight (xs : List α) (n : Nat := 1) : List α :=
-  let len := xs.length
-  if len ≤ 1 then
-    xs
-  else
-    let n := len - n % len
-    let b := xs.take n
-    let e := xs.drop n
-    e ++ b
+Monadic generalization of `List.partition`.
 
-theorem get_append_left (as bs : List α) (h : i < as.length) {h'} : (as ++ bs).get ⟨i, h'⟩ = as.get ⟨i, h⟩ := by
+This uses `Array.toList` and which isn't imported by `Init.Data.List.Basic` or `Init.Data.List.Control`.
+```
+def posOrNeg (x : Int) : Except String Bool :=
+  if x > 0 then pure true
+  else if x < 0 then pure false
+  else throw "Zero is not positive or negative"
+
+partitionM posOrNeg [-1, 2, 3] = Except.ok ([2, 3], [-1])
+partitionM posOrNeg [0, 2, 3] = Except.error "Zero is not positive or negative"
+```
+-/
+@[inline] def partitionM [Monad m] (p : α → m Bool) (l : List α) : m (List α × List α) :=
+  go l #[] #[]
+where
+  /-- Auxiliary for `partitionM`:
+  `partitionM.go p l acc₁ acc₂` returns `(acc₁.toList ++ left, acc₂.toList ++ right)`
+  if `partitionM p l` returns `(left, right)`. -/
+  @[specialize] go : List α → Array α → Array α → m (List α × List α)
+  | [], acc₁, acc₂ => pure (acc₁.toList, acc₂.toList)
+  | x :: xs, acc₁, acc₂ => do
+    if ← p x then
+      go xs (acc₁.push x) acc₂
+    else
+      go xs acc₁ (acc₂.push x)
+
+/-! ### partitionMap -/
+
+/--
+Given a function `f : α → β ⊕ γ`, `partitionMap f l` maps the list by `f`
+whilst partitioning the result into a pair of lists, `List β × List γ`,
+partitioning the `.inl _` into the left list, and the `.inr _` into the right List.
+```
+partitionMap (id : Nat ⊕ Nat → Nat ⊕ Nat) [inl 0, inr 1, inl 2] = ([0, 2], [1])
+```
+-/
+@[inline] def partitionMap (f : α → β ⊕ γ) (l : List α) : List β × List γ := go l #[] #[] where
+  /-- Auxiliary for `partitionMap`:
+  `partitionMap.go f l acc₁ acc₂ = (acc₁.toList ++ left, acc₂.toList ++ right)`
+  if `partitionMap f l = (left, right)`. -/
+  @[specialize] go : List α → Array β → Array γ → List β × List γ
+  | [], acc₁, acc₂ => (acc₁.toList, acc₂.toList)
+  | x :: xs, acc₁, acc₂ =>
+    match f x with
+    | .inl a => go xs (acc₁.push a) acc₂
+    | .inr b => go xs acc₁ (acc₂.push b)
+
+/-! ### mapMono
+
+This is a performance optimization for `List.mapM` that avoids allocating a new list when the result of each `f a` is a pointer equal value `a`.
+
+For verification purposes, `List.mapMono = List.map`.
+-/
+
+@[specialize] private unsafe def mapMonoMImp [Monad m] (as : List α) (f : α → m α) : m (List α) := do
+  match as with
+  | [] => return as
+  | b :: bs =>
+    let b'  ← f b
+    let bs' ← mapMonoMImp bs f
+    if ptrEq b' b && ptrEq bs' bs then
+      return as
+    else
+      return b' :: bs'
+
+/--
+Monomorphic `List.mapM`. The internal implementation uses pointer equality, and does not allocate a new list
+if the result of each `f a` is a pointer equal value `a`.
+-/
+@[implemented_by mapMonoMImp] def mapMonoM [Monad m] (as : List α) (f : α → m α) : m (List α) :=
+  match as with
+  | [] => return []
+  | a :: as => return (← f a) :: (← mapMonoM as f)
+
+def mapMono (as : List α) (f : α → α) : List α :=
+  Id.run <| as.mapMonoM f
+
+/-! ## Additional lemmas required for bootstrapping `Array`. -/
+
+theorem getElem_append_left (as bs : List α) (h : i < as.length) {h'} : (as ++ bs)[i] = as[i] := by
   induction as generalizing i with
   | nil => trivial
   | cons a as ih =>
@@ -199,7 +163,7 @@ theorem get_append_left (as bs : List α) (h : i < as.length) {h'} : (as ++ bs).
     | zero => rfl
     | succ i => apply ih
 
-theorem get_append_right (as bs : List α) (h : ¬ i < as.length) {h' h''} : (as ++ bs).get ⟨i, h'⟩ = bs.get ⟨i - as.length, h''⟩ := by
+theorem getElem_append_right (as bs : List α) (h : ¬ i < as.length) {h' h''} : (as ++ bs)[i]'h' = bs[i - as.length]'h'' := by
   induction as generalizing i with
   | nil => trivial
   | cons a as ih =>
@@ -285,74 +249,4 @@ theorem le_antisymm [LT α] [s : Antisymm (¬ · < · : α → α → Prop)] {as
 instance [LT α] [Antisymm (¬ · < · : α → α → Prop)] : Antisymm (· ≤ · : List α → List α → Prop) where
   antisymm h₁ h₂ := le_antisymm h₁ h₂
 
-@[specialize] private unsafe def mapMonoMImp [Monad m] (as : List α) (f : α → m α) : m (List α) := do
-  match as with
-  | [] => return as
-  | b :: bs =>
-    let b'  ← f b
-    let bs' ← mapMonoMImp bs f
-    if ptrEq b' b && ptrEq bs' bs then
-      return as
-    else
-      return b' :: bs'
-
-/--
-Monomorphic `List.mapM`. The internal implementation uses pointer equality, and does not allocate a new list
-if the result of each `f a` is a pointer equal value `a`.
--/
-@[implemented_by mapMonoMImp] def mapMonoM [Monad m] (as : List α) (f : α → m α) : m (List α) :=
-  match as with
-  | [] => return []
-  | a :: as => return (← f a) :: (← mapMonoM as f)
-
-def mapMono (as : List α) (f : α → α) : List α :=
-  Id.run <| as.mapMonoM f
-
-/--
-Monadic generalization of `List.partition`.
-
-This uses `Array.toList` and which isn't imported by `Init.Data.List.Basic`.
-```
-def posOrNeg (x : Int) : Except String Bool :=
-  if x > 0 then pure true
-  else if x < 0 then pure false
-  else throw "Zero is not positive or negative"
-
-partitionM posOrNeg [-1, 2, 3] = Except.ok ([2, 3], [-1])
-partitionM posOrNeg [0, 2, 3] = Except.error "Zero is not positive or negative"
-```
--/
-@[inline] def partitionM [Monad m] (p : α → m Bool) (l : List α) : m (List α × List α) :=
-  go l #[] #[]
-where
-  /-- Auxiliary for `partitionM`:
-  `partitionM.go p l acc₁ acc₂` returns `(acc₁.toList ++ left, acc₂.toList ++ right)`
-  if `partitionM p l` returns `(left, right)`. -/
-  @[specialize] go : List α → Array α → Array α → m (List α × List α)
-  | [], acc₁, acc₂ => pure (acc₁.toList, acc₂.toList)
-  | x :: xs, acc₁, acc₂ => do
-    if ← p x then
-      go xs (acc₁.push x) acc₂
-    else
-      go xs acc₁ (acc₂.push x)
-
-/--
-Given a function `f : α → β ⊕ γ`, `partitionMap f l` maps the list by `f`
-whilst partitioning the result it into a pair of lists, `List β × List γ`,
-partitioning the `.inl _` into the left list, and the `.inr _` into the right List.
-```
-partitionMap (id : Nat ⊕ Nat → Nat ⊕ Nat) [inl 0, inr 1, inl 2] = ([0, 2], [1])
-```
--/
-@[inline] def partitionMap (f : α → β ⊕ γ) (l : List α) : List β × List γ := go l #[] #[] where
-  /-- Auxiliary for `partitionMap`:
-  `partitionMap.go f l acc₁ acc₂ = (acc₁.toList ++ left, acc₂.toList ++ right)`
-  if `partitionMap f l = (left, right)`. -/
-  @[specialize] go : List α → Array β → Array γ → List β × List γ
-  | [], acc₁, acc₂ => (acc₁.toList, acc₂.toList)
-  | x :: xs, acc₁, acc₂ =>
-    match f x with
-    | .inl a => go xs (acc₁.push a) acc₂
-    | .inr b => go xs acc₁ (acc₂.push b)
-
 end List

src/Init/Data/List/Control.lean

@@ -151,6 +151,11 @@ protected def foldlM {m : Type u → Type v} [Monad m] {s : Type u} {α : Type w
     let s' ← f s a
     List.foldlM f s' as
 
+@[simp] theorem foldlM_nil [Monad m] (f : β → α → m β) (b) : [].foldlM f b = pure b := rfl
+@[simp] theorem foldlM_cons [Monad m] (f : β → α → m β) (b) (a) (l : List α) :
+    (a :: l).foldlM f b = f b a >>= l.foldlM f := by
+  simp [List.foldlM]
+
 /--
 Folds a monadic function over a list from right to left:
 ```
@@ -165,6 +170,8 @@ foldrM f x₀ [a, b, c] = do
 def foldrM {m : Type u → Type v} [Monad m] {s : Type u} {α : Type w} (f : α → s → m s) (init : s) (l : List α) : m s :=
   l.reverse.foldlM (fun s a => f a s) init
 
+@[simp] theorem foldrM_nil [Monad m] (f : α → β → m β) (b) : [].foldrM f b = pure b := rfl
+
 /--
 Maps `f` over the list and collects the results with `<|>`.
 ```

src/Init/Data/List/Impl.lean

@@ -16,7 +16,44 @@ so these are in a separate file to minimize imports.
 
 namespace List
 
-/-- Tail recursive version of `erase`. -/
+/-! ## Basic `List` operations.
+
+The following operations are already tail-recursive, and do not need `@[csimp]` replacements:
+`get`, `foldl`, `beq`, `isEqv`, `reverse`, `elem` (and hence `contains`), `drop`, `dropWhile`,
+`partition`, `isPrefixOf`, `isPrefixOf?`, `find?`, `findSome?`, `lookup`, `any` (and hence `or`),
+`all` (and hence `and`) , `range`, `eraseDups`, `eraseReps`, `span`, `groupBy`.
+
+The following operations are still missing `@[csimp]` replacements:
+`concat`, `zipWithAll`.
+
+The following operations are not recursive to begin with
+(or are defined in terms of recursive primitives):
+`isEmpty`, `isSuffixOf`, `isSuffixOf?`, `rotateLeft`, `rotateRight`, `insert`, `zip`, `enum`,
+`minimum?`, `maximum?`, and `removeAll`.
+
+The following operations are given `@[csimp]` replacements below:
+`length`, `set`, `map`, `filter`, `filterMap`, `foldr`, `append`, `bind`, `join`, `replicate`,
+`take`, `takeWhile`, `dropLast`, `replace`, `erase`, `eraseIdx`, `zipWith`, `unzip`, `iota`,
+`enumFrom`, `intersperse`, and `intercalate`.
+
+-/
+
+/-! ### length -/
+
+theorem length_add_eq_lengthTRAux (as : List α) (n : Nat) : as.length + n = as.lengthTRAux n := by
+  induction as generalizing n with
+  | nil  => simp [length, lengthTRAux]
+  | cons a as ih =>
+    simp [length, lengthTRAux, ← ih, Nat.succ_add]
+    rfl
+
+@[csimp] theorem length_eq_lengthTR : @List.length = @List.lengthTR := by
+  apply funext; intro α; apply funext; intro as
+  simp [lengthTR, ← length_add_eq_lengthTRAux]
+
+/-! ### set -/
+
+/-- Tail recursive version of `List.set`. -/
 @[inline] def setTR (l : List α) (n : Nat) (a : α) : List α := go l n #[] where
   /-- Auxiliary for `setTR`: `setTR.go l a xs n acc = acc.toList ++ set xs a`,
   unless `n ≥ l.length` in which case it returns `l` -/
@@ -31,10 +68,214 @@ namespace List
     setTR.go l a xs n acc = acc.data ++ xs.set n a
   | [], _ => fun h => by simp [setTR.go, set, h]
   | x::xs, 0 => by simp [setTR.go, set]
-  | x::xs, n+1 => fun h => by simp [setTR.go, set]; rw [go _ xs]; {simp}; simp [h]
+  | x::xs, n+1 => fun h => by simp only [setTR.go, set]; rw [go _ xs] <;> simp [h]
   exact (go #[] _ _ rfl).symm
 
-/-- Tail recursive version of `erase`. -/
+/-! ### map -/
+
+/-- Tail-recursive version of `List.map`. -/
+@[inline] def mapTR (f : α → β) (as : List α) : List β :=
+  loop as []
+where
+  @[specialize] loop : List α → List β → List β
+  | [],    bs => bs.reverse
+  | a::as, bs => loop as (f a :: bs)
+
+theorem mapTR_loop_eq (f : α → β) (as : List α) (bs : List β) :
+    mapTR.loop f as bs = bs.reverse ++ map f as := by
+  induction as generalizing bs with
+  | nil => simp [mapTR.loop, map]
+  | cons a as ih =>
+    simp only [mapTR.loop, map]
+    rw [ih (f a :: bs), reverse_cons, append_assoc]
+    rfl
+
+@[csimp] theorem map_eq_mapTR : @map = @mapTR :=
+  funext fun α => funext fun β => funext fun f => funext fun as => by
+    simp [mapTR, mapTR_loop_eq]
+
+/-! ### filter -/
+
+/-- Tail-recursive version of `List.filter`. -/
+@[inline] def filterTR (p : α → Bool) (as : List α) : List α :=
+  loop as []
+where
+  @[specialize] loop : List α → List α → List α
+  | [],    rs => rs.reverse
+  | a::as, rs => match p a with
+     | true  => loop as (a::rs)
+     | false => loop as rs
+
+theorem filterTR_loop_eq (p : α → Bool) (as bs : List α) :
+    filterTR.loop p as bs = bs.reverse ++ filter p as := by
+  induction as generalizing bs with
+  | nil => simp [filterTR.loop, filter]
+  | cons a as ih =>
+    simp only [filterTR.loop, filter]
+    split <;> simp_all
+
+@[csimp] theorem filter_eq_filterTR : @filter = @filterTR := by
+  apply funext; intro α; apply funext; intro p; apply funext; intro as
+  simp [filterTR, filterTR_loop_eq]
+
+/-! ### filterMap -/
+
+/-- Tail recursive version of `filterMap`. -/
+@[inline] def filterMapTR (f : α → Option β) (l : List α) : List β := go l #[] where
+  /-- Auxiliary for `filterMap`: `filterMap.go f l = acc.toList ++ filterMap f l` -/
+  @[specialize] go : List α → Array β → List β
+  | [], acc => acc.toList
+  | a::as, acc => match f a with
+    | none => go as acc
+    | some b => go as (acc.push b)
+
+@[csimp] theorem filterMap_eq_filterMapTR : @List.filterMap = @filterMapTR := by
+  funext α β f l
+  let rec go : ∀ as acc, filterMapTR.go f as acc = acc.data ++ as.filterMap f
+    | [], acc => by simp [filterMapTR.go, filterMap]
+    | a::as, acc => by
+      simp only [filterMapTR.go, go as, Array.push_data, append_assoc, singleton_append, filterMap]
+      split <;> simp [*]
+  exact (go l #[]).symm
+
+/-! ### foldr -/
+
+/-- Tail recursive version of `List.foldr`. -/
+@[specialize] def foldrTR (f : α → β → β) (init : β) (l : List α) : β := l.toArray.foldr f init
+
+@[csimp] theorem foldr_eq_foldrTR : @foldr = @foldrTR := by
+  funext α β f init l; simp [foldrTR, Array.foldr_eq_foldr_data, -Array.size_toArray]
+
+/-! ### bind  -/
+
+/-- Tail recursive version of `List.bind`. -/
+@[inline] def bindTR (as : List α) (f : α → List β) : List β := go as #[] where
+  /-- Auxiliary for `bind`: `bind.go f as = acc.toList ++ bind f as` -/
+  @[specialize] go : List α → Array β → List β
+  | [], acc => acc.toList
+  | x::xs, acc => go xs (acc ++ f x)
+
+@[csimp] theorem bind_eq_bindTR : @List.bind = @bindTR := by
+  funext α β as f
+  let rec go : ∀ as acc, bindTR.go f as acc = acc.data ++ as.bind f
+    | [], acc => by simp [bindTR.go, bind]
+    | x::xs, acc => by simp [bindTR.go, bind, go xs]
+  exact (go as #[]).symm
+
+/-! ### join -/
+
+/-- Tail recursive version of `List.join`. -/
+@[inline] def joinTR (l : List (List α)) : List α := bindTR l id
+
+@[csimp] theorem join_eq_joinTR : @join = @joinTR := by
+  funext α l; rw [← List.bind_id, List.bind_eq_bindTR]; rfl
+
+/-! ### replicate -/
+
+/-- Tail-recursive version of `List.replicate`. -/
+def replicateTR {α : Type u} (n : Nat) (a : α) : List α :=
+  let rec loop : Nat → List α → List α
+    | 0, as => as
+    | n+1, as => loop n (a::as)
+  loop n []
+
+theorem replicateTR_loop_replicate_eq (a : α) (m n : Nat) :
+  replicateTR.loop a n (replicate m a) = replicate (n + m) a := by
+  induction n generalizing m with simp [replicateTR.loop]
+  | succ n ih => simp [Nat.succ_add]; exact ih (m+1)
+
+theorem replicateTR_loop_eq : ∀ n, replicateTR.loop a n acc = replicate n a ++ acc
+  | 0 => rfl
+  | n+1 => by rw [← replicateTR_loop_replicate_eq _ 1 n, replicate, replicate,
+    replicateTR.loop, replicateTR_loop_eq n, replicateTR_loop_eq n, append_assoc]; rfl
+
+@[csimp] theorem replicate_eq_replicateTR : @List.replicate = @List.replicateTR := by
+  apply funext; intro α; apply funext; intro n; apply funext; intro a
+  exact (replicateTR_loop_replicate_eq _ 0 n).symm
+
+/-! ## Sublists -/
+
+/-! ### take -/
+
+/-- Tail recursive version of `List.take`. -/
+@[inline] def takeTR (n : Nat) (l : List α) : List α := go l n #[] where
+  /-- Auxiliary for `take`: `take.go l xs n acc = acc.toList ++ take n xs`,
+  unless `n ≥ xs.length` in which case it returns `l`. -/
+  @[specialize] go : List α → Nat → Array α → List α
+  | [], _, _ => l
+  | _::_, 0, acc => acc.toList
+  | a::as, n+1, acc => go as n (acc.push a)
+
+@[csimp] theorem take_eq_takeTR : @take = @takeTR := by
+  funext α n l; simp [takeTR]
+  suffices ∀ xs acc, l = acc.data ++ xs → takeTR.go l xs n acc = acc.data ++ xs.take n from
+    (this l #[] (by simp)).symm
+  intro xs; induction xs generalizing n with intro acc
+  | nil => cases n <;> simp [take, takeTR.go]
+  | cons x xs IH =>
+    cases n with simp only [take, takeTR.go]
+    | zero => simp
+    | succ n => intro h; rw [IH] <;> simp_all
+
+/-! ### takeWhile -/
+
+/-- Tail recursive version of `List.takeWhile`. -/
+@[inline] def takeWhileTR (p : α → Bool) (l : List α) : List α := go l #[] where
+  /-- Auxiliary for `takeWhile`: `takeWhile.go p l xs acc = acc.toList ++ takeWhile p xs`,
+  unless no element satisfying `p` is found in `xs` in which case it returns `l`. -/
+  @[specialize] go : List α → Array α → List α
+  | [], _ => l
+  | a::as, acc => bif p a then go as (acc.push a) else acc.toList
+
+@[csimp] theorem takeWhile_eq_takeWhileTR : @takeWhile = @takeWhileTR := by
+  funext α p l; simp [takeWhileTR]
+  suffices ∀ xs acc, l = acc.data ++ xs →
+      takeWhileTR.go p l xs acc = acc.data ++ xs.takeWhile p from
+    (this l #[] (by simp)).symm
+  intro xs; induction xs with intro acc
+  | nil => simp [takeWhile, takeWhileTR.go]
+  | cons x xs IH =>
+    simp only [takeWhileTR.go, Array.toList_eq, takeWhile]
+    split
+    · intro h; rw [IH] <;> simp_all
+    · simp [*]
+
+/-! ### dropLast -/
+
+/-- Tail recursive version of `dropLast`. -/
+@[inline] def dropLastTR (l : List α) : List α := l.toArray.pop.toList
+
+@[csimp] theorem dropLast_eq_dropLastTR : @dropLast = @dropLastTR := by
+  funext α l; simp [dropLastTR]
+
+/-! ## Manipulating elements -/
+
+/-! ### replace -/
+
+/-- Tail recursive version of `List.replace`. -/
+@[inline] def replaceTR [BEq α] (l : List α) (b c : α) : List α := go l #[] where
+  /-- Auxiliary for `replace`: `replace.go l b c xs acc = acc.toList ++ replace xs b c`,
+  unless `b` is not found in `xs` in which case it returns `l`. -/
+  @[specialize] go : List α → Array α → List α
+  | [], _ => l
+  | a::as, acc => bif b == a then acc.toListAppend (c::as) else go as (acc.push a)
+
+@[csimp] theorem replace_eq_replaceTR : @List.replace = @replaceTR := by
+  funext α _ l b c; simp [replaceTR]
+  suffices ∀ xs acc, l = acc.data ++ xs →
+      replaceTR.go l b c xs acc = acc.data ++ xs.replace b c from
+    (this l #[] (by simp)).symm
+  intro xs; induction xs with intro acc
+  | nil => simp [replace, replaceTR.go]
+  | cons x xs IH =>
+    simp only [replaceTR.go, Array.toListAppend_eq, replace]
+    split
+    · simp [*]
+    · intro h; rw [IH] <;> simp_all
+
+/-! ### erase -/
+
+/-- Tail recursive version of `List.erase`. -/
 @[inline] def eraseTR [BEq α] (l : List α) (a : α) : List α := go l #[] where
   /-- Auxiliary for `eraseTR`: `eraseTR.go l a xs acc = acc.toList ++ erase xs a`,
   unless `a` is not present in which case it returns `l` -/
@@ -49,11 +290,14 @@ namespace List
   intro xs; induction xs with intro acc h
   | nil => simp [List.erase, eraseTR.go, h]
   | cons x xs IH =>
-    simp [List.erase, eraseTR.go]
-    cases x == a <;> simp
-    · rw [IH]; simp; simp; exact h
+    simp only [eraseTR.go, Array.toListAppend_eq, List.erase]
+    cases x == a
+    · rw [IH] <;> simp_all
+    · simp
 
-/-- Tail recursive version of `eraseIdx`. -/
+/-! ### eraseIdx -/
+
+/-- Tail recursive version of `List.eraseIdx`. -/
 @[inline] def eraseIdxTR (l : List α) (n : Nat) : List α := go l n #[] where
   /-- Auxiliary for `eraseIdxTR`: `eraseIdxTR.go l n xs acc = acc.toList ++ eraseIdx xs a`,
   unless `a` is not present in which case it returns `l` -/
@@ -72,109 +316,14 @@ namespace List
     match n with
     | 0 => simp [eraseIdx, eraseIdxTR.go]
     | n+1 =>
-      simp [eraseIdx, eraseIdxTR.go]
+      simp only [eraseIdxTR.go, eraseIdx]
       rw [IH]; simp; simp; exact h
 
-/-- Tail recursive version of `bind`. -/
-@[inline] def bindTR (as : List α) (f : α → List β) : List β := go as #[] where
-  /-- Auxiliary for `bind`: `bind.go f as = acc.toList ++ bind f as` -/
-  @[specialize] go : List α → Array β → List β
-  | [], acc => acc.toList
-  | x::xs, acc => go xs (acc ++ f x)
+/-! ## Zippers -/
 
-@[csimp] theorem bind_eq_bindTR : @List.bind = @bindTR := by
-  funext α β as f
-  let rec go : ∀ as acc, bindTR.go f as acc = acc.data ++ as.bind f
-    | [], acc => by simp [bindTR.go, bind]
-    | x::xs, acc => by simp [bindTR.go, bind, go xs]
-  exact (go as #[]).symm
+/-! ### zipWith -/
 
-/-- Tail recursive version of `join`. -/
-@[inline] def joinTR (l : List (List α)) : List α := bindTR l id
-
-@[csimp] theorem join_eq_joinTR : @join = @joinTR := by
-  funext α l; rw [← List.bind_id, List.bind_eq_bindTR]; rfl
-
-/-- Tail recursive version of `filterMap`. -/
-@[inline] def filterMapTR (f : α → Option β) (l : List α) : List β := go l #[] where
-  /-- Auxiliary for `filterMap`: `filterMap.go f l = acc.toList ++ filterMap f l` -/
-  @[specialize] go : List α → Array β → List β
-  | [], acc => acc.toList
-  | a::as, acc => match f a with
-    | none => go as acc
-    | some b => go as (acc.push b)
-
-@[csimp] theorem filterMap_eq_filterMapTR : @List.filterMap = @filterMapTR := by
-  funext α β f l
-  let rec go : ∀ as acc, filterMapTR.go f as acc = acc.data ++ as.filterMap f
-    | [], acc => by simp [filterMapTR.go, filterMap]
-    | a::as, acc => by simp [filterMapTR.go, filterMap, go as]; split <;> simp [*]
-  exact (go l #[]).symm
-
-/-- Tail recursive version of `replace`. -/
-@[inline] def replaceTR [BEq α] (l : List α) (b c : α) : List α := go l #[] where
-  /-- Auxiliary for `replace`: `replace.go l b c xs acc = acc.toList ++ replace xs b c`,
-  unless `b` is not found in `xs` in which case it returns `l`. -/
-  @[specialize] go : List α → Array α → List α
-  | [], _ => l
-  | a::as, acc => bif a == b then acc.toListAppend (c::as) else go as (acc.push a)
-
-@[csimp] theorem replace_eq_replaceTR : @List.replace = @replaceTR := by
-  funext α _ l b c; simp [replaceTR]
-  suffices ∀ xs acc, l = acc.data ++ xs →
-      replaceTR.go l b c xs acc = acc.data ++ xs.replace b c from
-    (this l #[] (by simp)).symm
-  intro xs; induction xs with intro acc
-  | nil => simp [replace, replaceTR.go]
-  | cons x xs IH =>
-    simp [replace, replaceTR.go]; split <;> simp [*]
-    · intro h; rw [IH]; simp; simp; exact h
-
-/-- Tail recursive version of `take`. -/
-@[inline] def takeTR (n : Nat) (l : List α) : List α := go l n #[] where
-  /-- Auxiliary for `take`: `take.go l xs n acc = acc.toList ++ take n xs`,
-  unless `n ≥ xs.length` in which case it returns `l`. -/
-  @[specialize] go : List α → Nat → Array α → List α
-  | [], _, _ => l
-  | _::_, 0, acc => acc.toList
-  | a::as, n+1, acc => go as n (acc.push a)
-
-@[csimp] theorem take_eq_takeTR : @take = @takeTR := by
-  funext α n l; simp [takeTR]
-  suffices ∀ xs acc, l = acc.data ++ xs → takeTR.go l xs n acc = acc.data ++ xs.take n from
-    (this l #[] (by simp)).symm
-  intro xs; induction xs generalizing n with intro acc
-  | nil => cases n <;> simp [take, takeTR.go]
-  | cons x xs IH =>
-    cases n with simp [take, takeTR.go]
-    | succ n => intro h; rw [IH]; simp; simp; exact h
-
-/-- Tail recursive version of `takeWhile`. -/
-@[inline] def takeWhileTR (p : α → Bool) (l : List α) : List α := go l #[] where
-  /-- Auxiliary for `takeWhile`: `takeWhile.go p l xs acc = acc.toList ++ takeWhile p xs`,
-  unless no element satisfying `p` is found in `xs` in which case it returns `l`. -/
-  @[specialize] go : List α → Array α → List α
-  | [], _ => l
-  | a::as, acc => bif p a then go as (acc.push a) else acc.toList
-
-@[csimp] theorem takeWhile_eq_takeWhileTR : @takeWhile = @takeWhileTR := by
-  funext α p l; simp [takeWhileTR]
-  suffices ∀ xs acc, l = acc.data ++ xs →
-      takeWhileTR.go p l xs acc = acc.data ++ xs.takeWhile p from
-    (this l #[] (by simp)).symm
-  intro xs; induction xs with intro acc
-  | nil => simp [takeWhile, takeWhileTR.go]
-  | cons x xs IH =>
-    simp [takeWhile, takeWhileTR.go]; split <;> simp [*]
-    · intro h; rw [IH]; simp; simp; exact h
-
-/-- Tail recursive version of `foldr`. -/
-@[specialize] def foldrTR (f : α → β → β) (init : β) (l : List α) : β := l.toArray.foldr f init
-
-@[csimp] theorem foldr_eq_foldrTR : @foldr = @foldrTR := by
-  funext α β f init l; simp [foldrTR, Array.foldr_eq_foldr_data, -Array.size_toArray]
-
-/-- Tail recursive version of `zipWith`. -/
+/-- Tail recursive version of `List.zipWith`. -/
 @[inline] def zipWithTR (f : α → β → γ) (as : List α) (bs : List β) : List γ := go as bs #[] where
   /-- Auxiliary for `zipWith`: `zipWith.go f as bs acc = acc.toList ++ zipWith f as bs` -/
   go : List α → List β → Array γ → List γ
@@ -188,14 +337,37 @@ namespace List
     | a::as, b::bs, acc => by simp [zipWithTR.go, zipWith, go as bs]
   exact (go as bs #[]).symm
 
-/-- Tail recursive version of `unzip`. -/
+/-! ### unzip -/
+
+/-- Tail recursive version of `List.unzip`. -/
 def unzipTR (l : List (α × β)) : List α × List β :=
   l.foldr (fun (a, b) (al, bl) => (a::al, b::bl)) ([], [])
 
 @[csimp] theorem unzip_eq_unzipTR : @unzip = @unzipTR := by
   funext α β l; simp [unzipTR]; induction l <;> simp [*]
 
-/-- Tail recursive version of `enumFrom`. -/
+/-! ## Ranges and enumeration -/
+
+/-! ### iota -/
+
+/-- Tail-recursive version of `List.iota`. -/
+def iotaTR (n : Nat) : List Nat :=
+  let rec go : Nat → List Nat → List Nat
+    | 0, r => r.reverse
+    | m@(n+1), r => go n (m::r)
+  go n []
+
+@[csimp]
+theorem iota_eq_iotaTR : @iota = @iotaTR :=
+  have aux (n : Nat) (r : List Nat) : iotaTR.go n r = r.reverse ++ iota n := by
+    induction n generalizing r with
+    | zero => simp [iota, iotaTR.go]
+    | succ n ih => simp [iota, iotaTR.go, ih, append_assoc]
+  funext fun n => by simp [iotaTR, aux]
+
+/-! ### enumFrom -/
+
+/-- Tail recursive version of `List.enumFrom`. -/
 def enumFromTR (n : Nat) (l : List α) : List (Nat × α) :=
   let arr := l.toArray
   (arr.foldr (fun a (n, acc) => (n-1, (n-1, a) :: acc)) (n + arr.size, [])).2
@@ -211,18 +383,11 @@ def enumFromTR (n : Nat) (l : List α) : List (Nat × α) :=
   rw [Array.foldr_eq_foldr_data]
   simp [go]
 
-theorem replicateTR_loop_eq : ∀ n, replicateTR.loop a n acc = replicate n a ++ acc
-  | 0 => rfl
-  | n+1 => by rw [← replicateTR_loop_replicate_eq _ 1 n, replicate, replicate,
-    replicateTR.loop, replicateTR_loop_eq n, replicateTR_loop_eq n, append_assoc]; rfl
+/-! ## Other list operations -/
 
-/-- Tail recursive version of `dropLast`. -/
-@[inline] def dropLastTR (l : List α) : List α := l.toArray.pop.toList
+/-! ### intersperse -/
 
-@[csimp] theorem dropLast_eq_dropLastTR : @dropLast = @dropLastTR := by
-  funext α l; simp [dropLastTR]
-
-/-- Tail recursive version of `intersperse`. -/
+/-- Tail recursive version of `List.intersperse`. -/
 def intersperseTR (sep : α) : List α → List α
   | [] => []
   | [x] => [x]
@@ -234,7 +399,9 @@ def intersperseTR (sep : α) : List α → List α
   | [] | [_] => rfl
   | x::y::xs => simp [intersperse]; induction xs generalizing y <;> simp [*]
 
-/-- Tail recursive version of `intercalate`. -/
+/-! ### intercalate -/
+
+/-- Tail recursive version of `List.intercalate`. -/
 def intercalateTR (sep : List α) : List (List α) → List α
   | [] => []
   | [x] => x

src/Init/Data/List/Notation.lean

@@ -0,0 +1,53 @@
+/-
+Copyright (c) 2016 Microsoft Corporation. All rights reserved.
+Released under Apache 2.0 license as described in the file LICENSE.
+Author: Leonardo de Moura
+-/
+prelude
+import Init.Data.Nat.Div
+
+/-!
+# Notation for `List` literals.
+-/
+
+set_option linter.missingDocs true -- keep it documented
+open Decidable List
+
+/--
+The syntax `[a, b, c]` is shorthand for `a :: b :: c :: []`, or
+`List.cons a (List.cons b (List.cons c List.nil))`. It allows conveniently constructing
+list literals.
+
+For lists of length at least 64, an alternative desugaring strategy is used
+which uses let bindings as intermediates as in
+`let left := [d, e, f]; a :: b :: c :: left` to avoid creating very deep expressions.
+Note that this changes the order of evaluation, although it should not be observable
+unless you use side effecting operations like `dbg_trace`.
+-/
+syntax "[" withoutPosition(term,*,?) "]"  : term
+
+/--
+Auxiliary syntax for implementing `[$elem,*]` list literal syntax.
+The syntax `%[a,b,c|tail]` constructs a value equivalent to `a::b::c::tail`.
+It uses binary partitioning to construct a tree of intermediate let bindings as in
+`let left := [d, e, f]; a :: b :: c :: left` to avoid creating very deep expressions.
+-/
+syntax "%[" withoutPosition(term,*,? " | " term) "]" : term
+
+namespace Lean
+
+macro_rules
+  | `([ $elems,* ]) => do
+    -- NOTE: we do not have `TSepArray.getElems` yet at this point
+    let rec expandListLit (i : Nat) (skip : Bool) (result : TSyntax `term) : MacroM Syntax := do
+      match i, skip with
+      | 0,   _     => pure result
+      | i+1, true  => expandListLit i false result
+      | i+1, false => expandListLit i true  (← ``(List.cons $(⟨elems.elemsAndSeps.get! i⟩) $result))
+    let size := elems.elemsAndSeps.size
+    if size < 64 then
+      expandListLit size (size % 2 == 0) (← ``(List.nil))
+    else
+      `(%[ $elems,* | List.nil ])
+
+end Lean

src/Init/Data/List/TakeDrop.lean

@@ -8,10 +8,10 @@ import Init.Data.List.Lemmas
 import Init.Data.Nat.Lemmas
 
 /-!
-# Lemmas about `List.take`, `List.drop`, `List.zip` and `List.zipWith`.
+# Further lemmas about `List.take`, `List.drop`, `List.zip` and `List.zipWith`.
 
 These are in a separate file from most of the list lemmas
-as they required importing more lemmas about natural numbers.
+as they required importing more lemmas about natural numbers, and use `omega`.
 -/
 
 namespace List
@@ -20,8 +20,6 @@ open Nat
 
 /-! ### take -/
 
-abbrev take_succ_cons := @take_cons_succ
-
 @[simp] theorem length_take : ∀ (i : Nat) (l : List α), length (take i l) = min i (length l)
   | 0, l => by simp [Nat.zero_min]
   | succ n, [] => by simp [Nat.min_zero]
@@ -34,17 +32,6 @@ theorem length_take_le' (n) (l : List α) : length (take n l) ≤ l.length :=
 
 theorem length_take_of_le (h : n ≤ length l) : length (take n l) = n := by simp [Nat.min_eq_left h]
 
-theorem take_all_of_le {n} {l : List α} (h : length l ≤ n) : take n l = l :=
-  take_length_le h
-
-@[simp]
-theorem take_left : ∀ l₁ l₂ : List α, take (length l₁) (l₁ ++ l₂) = l₁
-  | [], _ => rfl
-  | a :: l₁, l₂ => congrArg (cons a) (take_left l₁ l₂)
-
-theorem take_left' {l₁ l₂ : List α} {n} (h : length l₁ = n) : take n (l₁ ++ l₂) = l₁ := by
-  rw [← h]; apply take_left
-
 theorem take_take : ∀ (n m) (l : List α), take n (take m l) = take (min n m) l
   | n, 0, l => by rw [Nat.min_zero, take_zero, take_nil]
   | 0, m, l => by rw [Nat.zero_min, take_zero, take_zero]
@@ -52,16 +39,15 @@ theorem take_take : ∀ (n m) (l : List α), take n (take m l) = take (min n m)
   | succ n, succ m, a :: l => by
     simp only [take, succ_min_succ, take_take n m l]
 
-theorem take_replicate (a : α) : ∀ n m : Nat, take n (replicate m a) = replicate (min n m) a
+@[simp] theorem take_replicate (a : α) : ∀ n m : Nat, take n (replicate m a) = replicate (min n m) a
   | n, 0 => by simp [Nat.min_zero]
   | 0, m => by simp [Nat.zero_min]
-  | succ n, succ m => by simp [succ_min_succ, take_replicate]
+  | succ n, succ m => by simp [replicate_succ, succ_min_succ, take_replicate]
 
-theorem map_take (f : α → β) :
-    ∀ (L : List α) (i : Nat), (L.take i).map f = (L.map f).take i
-  | [], i => by simp
-  | _, 0 => by simp
-  | h :: t, n + 1 => by dsimp; rw [map_take f t n]
+@[simp] theorem drop_replicate (a : α) : ∀ n m : Nat, drop n (replicate m a) = replicate (m - n) a
+  | n, 0 => by simp
+  | 0, m => by simp
+  | succ n, succ m => by simp [replicate_succ, succ_sub_succ, drop_replicate]
 
 /-- Taking the first `n` elements in `l₁ ++ l₂` is the same as appending the first `n` elements
 of `l₁` to the first `n - l₁.length` elements of `l₂`. -/
@@ -88,55 +74,51 @@ theorem take_append {l₁ l₂ : List α} (i : Nat) :
 
 /-- The `i`-th element of a list coincides with the `i`-th element of any of its prefixes of
 length `> i`. Version designed to rewrite from the big list to the small list. -/
-theorem get_take (L : List α) {i j : Nat} (hi : i < L.length) (hj : i < j) :
-    get L ⟨i, hi⟩ = get (L.take j) ⟨i, length_take .. ▸ Nat.lt_min.mpr ⟨hj, hi⟩⟩ :=
-  get_of_eq (take_append_drop j L).symm _ ▸ get_append ..
+theorem getElem_take (L : List α) {i j : Nat} (hi : i < L.length) (hj : i < j) :
+    L[i] = (L.take j)[i]'(length_take .. ▸ Nat.lt_min.mpr ⟨hj, hi⟩) :=
+  getElem_of_eq (take_append_drop j L).symm _ ▸ getElem_append ..
 
 /-- The `i`-th element of a list coincides with the `i`-th element of any of its prefixes of
 length `> i`. Version designed to rewrite from the small list to the big list. -/
+theorem getElem_take' (L : List α) {j i : Nat} {h : i < (L.take j).length} :
+    (L.take j)[i] =
+    L[i]'(Nat.lt_of_lt_of_le h (length_take_le' _ _)) := by
+  rw [length_take, Nat.lt_min] at h; rw [getElem_take L _ h.1]
+
+/-- The `i`-th element of a list coincides with the `i`-th element of any of its prefixes of
+length `> i`. Version designed to rewrite from the big list to the small list. -/
+@[deprecated getElem_take (since := "2024-06-12")]
+theorem get_take (L : List α) {i j : Nat} (hi : i < L.length) (hj : i < j) :
+    get L ⟨i, hi⟩ = get (L.take j) ⟨i, length_take .. ▸ Nat.lt_min.mpr ⟨hj, hi⟩⟩ := by
+  simp [getElem_take _ hi hj]
+
+/-- The `i`-th element of a list coincides with the `i`-th element of any of its prefixes of
+length `> i`. Version designed to rewrite from the small list to the big list. -/
+@[deprecated getElem_take (since := "2024-06-12")]
 theorem get_take' (L : List α) {j i} :
     get (L.take j) i =
     get L ⟨i.1, Nat.lt_of_lt_of_le i.2 (length_take_le' _ _)⟩ := by
-  let ⟨i, hi⟩ := i; rw [length_take, Nat.lt_min] at hi; rw [get_take L _ hi.1]
+  simp [getElem_take']
 
-theorem get?_take {l : List α} {n m : Nat} (h : m < n) : (l.take n).get? m = l.get? m := by
-  induction n generalizing l m with
-  | zero =>
-    exact absurd h (Nat.not_lt_of_le m.zero_le)
-  | succ _ hn =>
-    cases l with
-    | nil => simp only [take_nil]
-    | cons hd tl =>
-      cases m
-      · simp only [get?, take]
-      · simpa only using hn (Nat.lt_of_succ_lt_succ h)
+theorem getElem?_take_eq_none {l : List α} {n m : Nat} (h : n ≤ m) :
+    (l.take n)[m]? = none :=
+  getElem?_eq_none <| Nat.le_trans (length_take_le _ _) h
 
+@[deprecated getElem?_take_eq_none (since := "2024-06-12")]
 theorem get?_take_eq_none {l : List α} {n m : Nat} (h : n ≤ m) :
-    (l.take n).get? m = none :=
-  get?_eq_none.mpr <| Nat.le_trans (length_take_le _ _) h
+    (l.take n).get? m = none := by
+  simp [getElem?_take_eq_none h]
 
+theorem getElem?_take_eq_if {l : List α} {n m : Nat} :
+    (l.take n)[m]? = if m < n then l[m]? else none := by
+  split
+  · next h => exact getElem?_take h
+  · next h => exact getElem?_take_eq_none (Nat.le_of_not_lt h)
+
+@[deprecated getElem?_take_eq_if (since := "2024-06-12")]
 theorem get?_take_eq_if {l : List α} {n m : Nat} :
     (l.take n).get? m = if m < n then l.get? m else none := by
-  split
-  · next h => exact get?_take h
-  · next h => exact get?_take_eq_none (Nat.le_of_not_lt h)
-
-@[simp]
-theorem nth_take_of_succ {l : List α} {n : Nat} : (l.take (n + 1)).get? n = l.get? n :=
-  get?_take (Nat.lt_succ_self n)
-
-theorem take_succ {l : List α} {n : Nat} : l.take (n + 1) = l.take n ++ (l.get? n).toList := by
-  induction l generalizing n with
-  | nil =>
-    simp only [Option.toList, get?, take_nil, append_nil]
-  | cons hd tl hl =>
-    cases n
-    · simp only [Option.toList, get?, eq_self_iff_true, take, nil_append]
-    · simp only [hl, cons_append, get?, eq_self_iff_true, take]
-
-@[simp]
-theorem take_eq_nil_iff {l : List α} {k : Nat} : l.take k = [] ↔ l = [] ∨ k = 0 := by
-  cases l <;> cases k <;> simp [Nat.succ_ne_zero]
+  simp [getElem?_take_eq_if]
 
 @[simp]
 theorem take_eq_take :
@@ -158,20 +140,6 @@ theorem take_add (l : List α) (m n : Nat) : l.take (m + n) = l.take m ++ (l.dro
   · apply length_take_le
   · apply Nat.le_add_right
 
-theorem take_eq_nil_of_eq_nil : ∀ {as : List α} {i}, as = [] → as.take i = []
-  | _, _, rfl => take_nil
-
-theorem ne_nil_of_take_ne_nil {as : List α} {i : Nat} (h: as.take i ≠ []) : as ≠ [] :=
-  mt take_eq_nil_of_eq_nil h
-
-theorem dropLast_eq_take (l : List α) : l.dropLast = l.take l.length.pred := by
-  cases l with
-  | nil => simp [dropLast]
-  | cons x l =>
-    induction l generalizing x with
-    | nil => simp [dropLast]
-    | cons hd tl hl => simp [dropLast, hl]
-
 theorem dropLast_take {n : Nat} {l : List α} (h : n < l.length) :
     (l.take n).dropLast = l.take n.pred := by
   simp only [dropLast_eq_take, length_take, Nat.le_of_lt h, take_take, pred_le, Nat.min_eq_left]
@@ -188,19 +156,6 @@ theorem map_eq_append_split {f : α → β} {l : List α} {s₁ s₂ : List β}
 
 /-! ### drop -/
 
-@[simp]
-theorem drop_eq_nil_iff_le {l : List α} {k : Nat} : l.drop k = [] ↔ l.length ≤ k := by
-  refine' ⟨fun h => _, drop_eq_nil_of_le⟩
-  induction k generalizing l with
-  | zero =>
-    simp only [drop] at h
-    simp [h]
-  | succ k hk =>
-    cases l
-    · simp
-    · simp only [drop] at h
-      simpa [Nat.succ_le_succ_iff] using hk h
-
 theorem drop_length_cons {l : List α} (h : l ≠ []) (a : α) :
     (a :: l).drop l.length = [l.getLast h] := by
   induction l generalizing a with
@@ -237,15 +192,6 @@ theorem drop_append {l₁ l₂ : List α} (i : Nat) : drop (l₁.length + i) (l
   rw [drop_append_eq_append_drop, drop_eq_nil_of_le] <;>
     simp [Nat.add_sub_cancel_left, Nat.le_add_right]
 
-theorem drop_sizeOf_le [SizeOf α] (l : List α) (n : Nat) : sizeOf (l.drop n) ≤ sizeOf l := by
-  induction l generalizing n with
-  | nil => rw [drop_nil]; apply Nat.le_refl
-  | cons _ _ lih =>
-    induction n with
-    | zero => apply Nat.le_refl
-    | succ n =>
-      exact Trans.trans (lih _) (Nat.le_add_left _ _)
-
 theorem lt_length_drop (L : List α) {i j : Nat} (h : i + j < L.length) : j < (L.drop i).length := by
   have A : i < L.length := Nat.lt_of_le_of_lt (Nat.le.intro rfl) h
   rw [(take_append_drop i L).symm] at h
@@ -254,24 +200,40 @@ theorem lt_length_drop (L : List α) {i j : Nat} (h : i + j < L.length) : j < (L
 
 /-- The `i + j`-th element of a list coincides with the `j`-th element of the list obtained by
 dropping the first `i` elements. Version designed to rewrite from the big list to the small list. -/
-theorem get_drop (L : List α) {i j : Nat} (h : i + j < L.length) :
-    get L ⟨i + j, h⟩ = get (L.drop i) ⟨j, lt_length_drop L h⟩ := by
+theorem getElem_drop (L : List α) {i j : Nat} (h : i + j < L.length) :
+    L[i + j] = (L.drop i)[j]'(lt_length_drop L h) := by
   have : i ≤ L.length := Nat.le_trans (Nat.le_add_right _ _) (Nat.le_of_lt h)
-  rw [get_of_eq (take_append_drop i L).symm ⟨i + j, h⟩, get_append_right'] <;>
+  rw [getElem_of_eq (take_append_drop i L).symm h, getElem_append_right'] <;>
     simp [Nat.min_eq_left this, Nat.add_sub_cancel_left, Nat.le_add_right]
 
+/-- The `i + j`-th element of a list coincides with the `j`-th element of the list obtained by
+dropping the first `i` elements. Version designed to rewrite from the big list to the small list. -/
+@[deprecated getElem_drop (since := "2024-06-12")]
+theorem get_drop (L : List α) {i j : Nat} (h : i + j < L.length) :
+    get L ⟨i + j, h⟩ = get (L.drop i) ⟨j, lt_length_drop L h⟩ := by
+  simp [getElem_drop]
+
 /-- The `i + j`-th element of a list coincides with the `j`-th element of the list obtained by
 dropping the first `i` elements. Version designed to rewrite from the small list to the big list. -/
+theorem getElem_drop' (L : List α) {i : Nat} {j : Nat} {h : j < (L.drop i).length} :
+    (L.drop i)[j] = L[i + j]'(by
+      rw [Nat.add_comm]
+      exact Nat.add_lt_of_lt_sub (length_drop i L ▸ h)) := by
+  rw [getElem_drop]
+
+/-- The `i + j`-th element of a list coincides with the `j`-th element of the list obtained by
+dropping the first `i` elements. Version designed to rewrite from the small list to the big list. -/
+@[deprecated getElem_drop' (since := "2024-06-12")]
 theorem get_drop' (L : List α) {i j} :
     get (L.drop i) j = get L ⟨i + j, by
       rw [Nat.add_comm]
       exact Nat.add_lt_of_lt_sub (length_drop i L ▸ j.2)⟩ := by
-  rw [get_drop]
+  simp [getElem_drop']
 
 @[simp]
-theorem get?_drop (L : List α) (i j : Nat) : get? (L.drop i) j = get? L (i + j) := by
+theorem getElem?_drop (L : List α) (i j : Nat) : (L.drop i)[j]? = L[i + j]? := by
   ext
-  simp only [get?_eq_some, get_drop', Option.mem_def]
+  simp only [getElem?_eq_some, getElem_drop', Option.mem_def]
   constructor <;> intro ⟨h, ha⟩
   · exact ⟨_, ha⟩
   · refine ⟨?_, ha⟩
@@ -279,19 +241,36 @@ theorem get?_drop (L : List α) (i j : Nat) : get? (L.drop i) j = get? L (i + j)
     rw [Nat.add_comm] at h
     apply Nat.lt_sub_of_add_lt h
 
-@[simp] theorem drop_drop (n : Nat) : ∀ (m) (l : List α), drop n (drop m l) = drop (n + m) l
-  | m, [] => by simp
-  | 0, l => by simp
-  | m + 1, a :: l =>
-    calc
-      drop n (drop (m + 1) (a :: l)) = drop n (drop m l) := rfl
-      _ = drop (n + m) l := drop_drop n m l
-      _ = drop (n + (m + 1)) (a :: l) := rfl
+@[deprecated getElem?_drop (since := "2024-06-12")]
+theorem get?_drop (L : List α) (i j : Nat) : get? (L.drop i) j = get? L (i + j) := by
+  simp
 
-theorem take_drop : ∀ (m n : Nat) (l : List α), take n (drop m l) = drop m (take (m + n) l)
-  | 0, _, _ => by simp
-  | _, _, [] => by simp
-  | _+1, _, _ :: _ => by simpa [Nat.succ_add, take_succ_cons, drop_succ_cons] using take_drop ..
+theorem set_eq_take_append_cons_drop {l : List α} {n : Nat} {a : α} :
+    l.set n a = if n < l.length then l.take n ++ a :: l.drop (n + 1) else l := by
+  split <;> rename_i h
+  · ext1 m
+    by_cases h' : m < n
+    · rw [getElem?_append (by simp [length_take]; omega), getElem?_set_ne (by omega),
+        getElem?_take h']
+    · by_cases h'' : m = n
+      · subst h''
+        rw [getElem?_set_eq (by simp; omega), getElem?_append_right, length_take,
+          Nat.min_eq_left (by omega), Nat.sub_self, getElem?_cons_zero]
+        rw [length_take]
+        exact Nat.min_le_left m l.length
+      · have h''' : n < m := by omega
+        rw [getElem?_set_ne (by omega), getElem?_append_right, length_take,
+          Nat.min_eq_left (by omega)]
+        · obtain ⟨k, rfl⟩ := Nat.exists_eq_add_of_lt h'''
+          have p : n + k + 1 - n = k + 1 := by omega
+          rw [p]
+          rw [getElem?_cons_succ, getElem?_drop]
+          congr 1
+          omega
+        · rw [length_take]
+          exact Nat.le_trans (Nat.min_le_left _ _) (by omega)
+  · rw [set_eq_of_length_le]
+    omega
 
 theorem drop_take : ∀ (m n : Nat) (l : List α), drop n (take m l) = take (m - n) (drop n l)
   | 0, _, _ => by simp
@@ -302,15 +281,7 @@ theorem drop_take : ∀ (m n : Nat) (l : List α), drop n (take m l) = take (m -
     congr 1
     omega
 
-theorem map_drop (f : α → β) :
-    ∀ (L : List α) (i : Nat), (L.drop i).map f = (L.map f).drop i
-  | [], i => by simp
-  | L, 0 => by simp
-  | h :: t, n + 1 => by
-    dsimp
-    rw [map_drop f t]
-
-theorem reverse_take {α} {xs : List α} (n : Nat) (h : n ≤ xs.length) :
+theorem take_reverse {α} {xs : List α} (n : Nat) (h : n ≤ xs.length) :
     xs.reverse.take n = (xs.drop (xs.length - n)).reverse := by
   induction xs generalizing n <;>
     simp only [reverse_cons, drop, reverse_nil, Nat.zero_sub, length, take_nil]
@@ -330,19 +301,33 @@ theorem reverse_take {α} {xs : List α} (n : Nat) (h : n ≤ xs.length) :
     rw [length_append, length_reverse]
     rfl
 
-@[simp]
-theorem get_cons_drop : ∀ (l : List α) i, get l i :: drop (i + 1) l = drop i l
-  | _::_, ⟨0, _⟩ => rfl
-  | _::_, ⟨i+1, _⟩ => get_cons_drop _ ⟨i, _⟩
+@[deprecated (since := "2024-06-15")] abbrev reverse_take := @take_reverse
 
-theorem drop_eq_get_cons {n} {l : List α} (h) : drop n l = get l ⟨n, h⟩ :: drop (n + 1) l :=
-  (get_cons_drop _ ⟨n, h⟩).symm
+/-! ### rotateLeft -/
 
-theorem drop_eq_nil_of_eq_nil : ∀ {as : List α} {i}, as = [] → as.drop i = []
-  | _, _, rfl => drop_nil
+@[simp] theorem rotateLeft_replicate (n) (a : α) : rotateLeft (replicate m a) n = replicate m a := by
+  cases n with
+  | zero => simp
+  | succ n =>
+    suffices 1 < m → m - (n + 1) % m + min ((n + 1) % m) m = m by
+      simpa [rotateLeft]
+    intro h
+    rw [Nat.min_eq_left (Nat.le_of_lt (Nat.mod_lt _ (by omega)))]
+    have : (n + 1) % m < m := Nat.mod_lt _ (by omega)
+    omega
 
-theorem ne_nil_of_drop_ne_nil {as : List α} {i : Nat} (h: as.drop i ≠ []) : as ≠ [] :=
-  mt drop_eq_nil_of_eq_nil h
+/-! ### rotateLeft -/
+
+@[simp] theorem rotateRight_replicate (n) (a : α) : rotateRight (replicate m a) n = replicate m a := by
+  cases n with
+  | zero => simp
+  | succ n =>
+    suffices 1 < m → m - (m - (n + 1) % m) + min (m - (n + 1) % m) m = m by
+      simpa [rotateRight]
+    intro h
+    have : (n + 1) % m < m := Nat.mod_lt _ (by omega)
+    rw [Nat.min_eq_left (by omega)]
+    omega
 
 /-! ### zipWith -/
 
@@ -351,10 +336,32 @@ theorem ne_nil_of_drop_ne_nil {as : List α} {i : Nat} (h: as.drop i ≠ []) : a
   induction l₁ generalizing l₂ <;> cases l₂ <;>
     simp_all [succ_min_succ, Nat.zero_min, Nat.min_zero]
 
+theorem zipWith_eq_zipWith_take_min : ∀ (l₁ : List α) (l₂ : List β),
+    zipWith f l₁ l₂ = zipWith f (l₁.take (min l₁.length l₂.length)) (l₂.take (min l₁.length l₂.length))
+  | [], _ => by simp
+  | _, [] => by simp
+  | a :: l₁, b :: l₂ => by simp [succ_min_succ, zipWith_eq_zipWith_take_min l₁ l₂]
+
+@[simp] theorem zipWith_replicate {a : α} {b : β} {m n : Nat} :
+    zipWith f (replicate m a) (replicate n b) = replicate (min m n) (f a b) := by
+  rw [zipWith_eq_zipWith_take_min]
+  simp
+
 /-! ### zip -/
 
 @[simp] theorem length_zip (l₁ : List α) (l₂ : List β) :
     length (zip l₁ l₂) = min (length l₁) (length l₂) := by
   simp [zip]
 
+theorem zip_eq_zip_take_min : ∀ (l₁ : List α) (l₂ : List β),
+    zip l₁ l₂ = zip (l₁.take (min l₁.length l₂.length)) (l₂.take (min l₁.length l₂.length))
+  | [], _ => by simp
+  | _, [] => by simp
+  | a :: l₁, b :: l₂ => by simp [succ_min_succ, zip_eq_zip_take_min l₁ l₂]
+
+@[simp] theorem zip_replicate {a : α} {b : β} {m n : Nat} :
+    zip (replicate m a) (replicate n b) = replicate (min m n) (a, b) := by
+  rw [zip_eq_zip_take_min]
+  simp
+
 end List

src/Init/Data/Nat/Basic.lean

@@ -200,6 +200,9 @@ protected theorem eq_zero_of_add_eq_zero_left (h : n + m = 0) : m = 0 :=
 theorem mul_succ (n m : Nat) : n * succ m = n * m + n :=
   rfl
 
+theorem mul_add_one (n m : Nat) : n * (m + 1) = n * m + n :=
+  rfl
+
 @[simp] protected theorem zero_mul : ∀ (n : Nat), 0 * n = 0
   | 0      => rfl
   | succ n => mul_succ 0 n ▸ (Nat.zero_mul n).symm ▸ rfl
@@ -209,6 +212,8 @@ theorem succ_mul (n m : Nat) : (succ n) * m = (n * m) + m := by
   | zero => rfl
   | succ m ih => rw [mul_succ, add_succ, ih, mul_succ, add_succ, Nat.add_right_comm]
 
+theorem add_one_mul (n m : Nat) : (n + 1) * m = (n * m) + m := succ_mul n m
+
 protected theorem mul_comm : ∀ (n m : Nat), n * m = m * n
   | n, 0      => (Nat.zero_mul n).symm ▸ (Nat.mul_zero n).symm ▸ rfl
   | n, succ m => (mul_succ n m).symm ▸ (succ_mul m n).symm ▸ (Nat.mul_comm n m).symm ▸ rfl
@@ -256,8 +261,18 @@ theorem succ_lt_succ {n m : Nat} : n < m → succ n < succ m := succ_le_succ
 
 theorem lt_succ_of_le {n m : Nat} : n ≤ m → n < succ m := succ_le_succ
 
+theorem le_of_lt_add_one {n m : Nat} : n < m + 1 → n ≤ m := le_of_succ_le_succ
+
+theorem lt_add_one_of_le {n m : Nat} : n ≤ m → n < m + 1 := succ_le_succ
+
 @[simp] protected theorem sub_zero (n : Nat) : n - 0 = n := rfl
 
+theorem not_add_one_le_zero (n : Nat) : ¬ n + 1 ≤ 0 := nofun
+
+theorem not_add_one_le_self : (n : Nat) → ¬ n + 1 ≤ n := Nat.not_succ_le_self
+
+theorem add_one_pos (n : Nat) : 0 < n + 1 := Nat.zero_lt_succ n
+
 theorem succ_sub_succ_eq_sub (n m : Nat) : succ n - succ m = n - m := by
   induction m with
   | zero      => exact rfl
@@ -271,6 +286,8 @@ theorem pred_lt : ∀ {n : Nat}, n ≠ 0 → pred n < n
   | zero,   h => absurd rfl h
   | succ _, _ => lt_succ_of_le (Nat.le_refl _)
 
+theorem sub_one_lt : ∀ {n : Nat}, n ≠ 0 → n - 1 < n := pred_lt
+
 theorem sub_le (n m : Nat) : n - m ≤ n := by
   induction m with
   | zero      => exact Nat.le_refl (n - 0)
@@ -340,6 +357,8 @@ theorem lt.base (n : Nat) : n < succ n := Nat.le_refl (succ n)
 
 @[simp] theorem lt_succ_self (n : Nat) : n < succ n := lt.base n
 
+@[simp] protected theorem lt_add_one (n : Nat) : n < n + 1 := lt.base n
+
 protected theorem le_total (m n : Nat) : m ≤ n ∨ n ≤ m :=
   match Nat.lt_or_ge m n with
   | Or.inl h => Or.inl (Nat.le_of_lt h)
@@ -370,6 +389,9 @@ theorem le_or_eq_of_le_succ {m n : Nat} (h : m ≤ succ n) : m ≤ n ∨ m = suc
        have : succ m ≤ succ n := succ_le_of_lt this
        Or.inl (le_of_succ_le_succ this))
 
+theorem le_or_eq_of_le_add_one {m n : Nat} (h : m ≤ n + 1) : m ≤ n ∨ m = n + 1 :=
+  le_or_eq_of_le_succ h
+
 theorem le_add_right : ∀ (n k : Nat), n ≤ n + k
   | n, 0   => Nat.le_refl n
   | n, k+1 => le_succ_of_le (le_add_right n k)
@@ -377,12 +399,25 @@ theorem le_add_right : ∀ (n k : Nat), n ≤ n + k
 theorem le_add_left (n m : Nat): n ≤ m + n :=
   Nat.add_comm n m ▸ le_add_right n m
 
+theorem le_of_add_right_le {n m k : Nat} (h : n + k ≤ m) : n ≤ m :=
+  Nat.le_trans (le_add_right n k) h
+
+theorem le_add_right_of_le {n m k : Nat} (h : n ≤ m) : n ≤ m + k :=
+  Nat.le_trans h (le_add_right m k)
+
+theorem lt_of_add_one_le {n m : Nat} (h : n + 1 ≤ m) : n < m := h
+
+theorem add_one_le_of_lt {n m : Nat} (h : n < m) : n + 1 ≤ m := h
+
 protected theorem lt_add_left (c : Nat) (h : a < b) : a < c + b :=
   Nat.lt_of_lt_of_le h (Nat.le_add_left ..)
 
 protected theorem lt_add_right (c : Nat) (h : a < b) : a < b + c :=
   Nat.lt_of_lt_of_le h (Nat.le_add_right ..)
 
+theorem lt_of_add_right_lt {n m k : Nat} (h : n + k < m) : n < m :=
+  Nat.lt_of_le_of_lt (Nat.le_add_right ..) h
+
 theorem le.dest : ∀ {n m : Nat}, n ≤ m → Exists (fun k => n + k = m)
   | zero,   zero,   _ => ⟨0, rfl⟩
   | zero,   succ n, _ => ⟨succ n, Nat.add_comm 0 (succ n) ▸ rfl⟩
@@ -537,9 +572,14 @@ protected theorem le_iff_lt_or_eq {n m : Nat} : n ≤ m ↔ n < m ∨ n = m :=
 
 protected theorem lt_succ_iff : m < succ n ↔ m ≤ n := ⟨le_of_lt_succ, lt_succ_of_le⟩
 
+protected theorem lt_add_one_iff : m < n + 1 ↔ m ≤ n := ⟨le_of_lt_succ, lt_succ_of_le⟩
+
 protected theorem lt_succ_iff_lt_or_eq : m < succ n ↔ m < n ∨ m = n :=
   Nat.lt_succ_iff.trans Nat.le_iff_lt_or_eq
 
+protected theorem lt_add_one_iff_lt_or_eq : m < n + 1 ↔ m < n ∨ m = n :=
+  Nat.lt_add_one_iff.trans Nat.le_iff_lt_or_eq
+
 protected theorem eq_of_lt_succ_of_not_lt (hmn : m < n + 1) (h : ¬ m < n) : m = n :=
   (Nat.lt_succ_iff_lt_or_eq.1 hmn).resolve_left h
 
@@ -571,12 +611,18 @@ attribute [simp] zero_lt_succ
 
 theorem succ_ne_self (n) : succ n ≠ n := Nat.ne_of_gt (lt_succ_self n)
 
+theorem add_one_ne_self (n) : n + 1 ≠ n := Nat.ne_of_gt (lt_succ_self n)
+
 theorem succ_le : succ n ≤ m ↔ n < m := .rfl
 
+theorem add_one_le_iff : n + 1 ≤ m ↔ n < m := .rfl
+
 theorem lt_succ : m < succ n ↔ m ≤ n := ⟨le_of_lt_succ, lt_succ_of_le⟩
 
 theorem lt_succ_of_lt (h : a < b) : a < succ b := le_succ_of_le h
 
+theorem lt_add_one_of_lt (h : a < b) : a < b + 1 := le_succ_of_le h
+
 theorem succ_pred_eq_of_ne_zero : ∀ {n}, n ≠ 0 → succ (pred n) = n
   | _+1, _ => rfl
 
@@ -590,12 +636,21 @@ theorem succ_le_succ_iff : succ a ≤ succ b ↔ a ≤ b := ⟨le_of_succ_le_suc
 
 theorem succ_lt_succ_iff : succ a < succ b ↔ a < b := ⟨lt_of_succ_lt_succ, succ_lt_succ⟩
 
+theorem add_one_inj : a + 1 = b + 1 ↔ a = b := succ_inj'
+
+theorem add_one_le_add_one_iff : a + 1 ≤ b + 1 ↔ a ≤ b := succ_le_succ_iff
+
+theorem add_one_lt_add_one_iff : a + 1 < b + 1 ↔ a < b := succ_lt_succ_iff
+
 theorem pred_inj : ∀ {a b}, 0 < a → 0 < b → pred a = pred b → a = b
   | _+1, _+1, _, _ => congrArg _
 
 theorem pred_ne_self : ∀ {a}, a ≠ 0 → pred a ≠ a
   | _+1, _ => (succ_ne_self _).symm
 
+theorem sub_one_ne_self : ∀ {a}, a ≠ 0 → a - 1 ≠ a
+  | _+1, _ => (succ_ne_self _).symm
+
 theorem pred_lt_self : ∀ {a}, 0 < a → pred a < a
   | _+1, _ => lt_succ_self _
 
@@ -628,9 +683,17 @@ theorem le_sub_one_of_lt : a < b → a ≤ b - 1 := Nat.le_pred_of_lt
 
 theorem lt_of_le_pred (h : 0 < m) : n ≤ pred m → n < m := (le_pred_iff_lt h).1
 
+theorem lt_of_le_sub_one (h : 0 < m) : n ≤ m - 1 → n < m := (le_pred_iff_lt h).1
+
+protected theorem le_sub_one_iff_lt (h : 0 < m) : n ≤ m - 1 ↔ n < m :=
+  ⟨Nat.lt_of_le_sub_one h, Nat.le_sub_one_of_lt⟩
+
 theorem exists_eq_succ_of_ne_zero : ∀ {n}, n ≠ 0 → Exists fun k => n = succ k
   | _+1, _ => ⟨_, rfl⟩
 
+theorem exists_eq_add_one_of_ne_zero : ∀ {n}, n ≠ 0 → Exists fun k => n = k + 1
+  | _+1, _ => ⟨_, rfl⟩
+
 /-! # Basic theorems for comparing numerals -/
 
 theorem ctor_eq_zero : Nat.zero = 0 :=
@@ -686,6 +749,9 @@ theorem eq_of_mul_eq_mul_right {n m k : Nat} (hm : 0 < m) (h : n * m = k * m) :
 protected theorem pow_succ (n m : Nat) : n^(succ m) = n^m * n :=
   rfl
 
+protected theorem pow_add_one (n m : Nat) : n^(m + 1) = n^m * n :=
+  rfl
+
 protected theorem pow_zero (n : Nat) : n^0 = 1 := rfl
 
 theorem pow_le_pow_of_le_left {n m : Nat} (h : n ≤ m) : ∀ (i : Nat), n^i ≤ m^i
@@ -737,9 +803,15 @@ theorem not_eq_zero_of_lt (h : b < a) : a ≠ 0 := by
   exact absurd h (Nat.not_lt_zero _)
   apply Nat.noConfusion
 
-theorem pred_lt' {n m : Nat} (h : m < n) : pred n < n :=
+theorem pred_lt_of_lt {n m : Nat} (h : m < n) : pred n < n :=
   pred_lt (not_eq_zero_of_lt h)
 
+set_option linter.missingDocs false in
+@[deprecated (since := "2024-06-01")] abbrev pred_lt' := @pred_lt_of_lt
+
+theorem sub_one_lt_of_lt {n m : Nat} (h : m < n) : n - 1 < n :=
+  sub_one_lt (not_eq_zero_of_lt h)
+
 /-! # pred theorems -/
 
 @[simp] protected theorem pred_zero : pred 0 = 0 := rfl
@@ -750,12 +822,21 @@ theorem succ_pred {a : Nat} (h : a ≠ 0) : a.pred.succ = a := by
   | zero => contradiction
   | succ => rfl
 
+theorem sub_one_add_one {a : Nat} (h : a ≠ 0) : a - 1 + 1 = a := by
+  induction a with
+  | zero => contradiction
+  | succ => rfl
+
 theorem succ_pred_eq_of_pos : ∀ {n}, 0 < n → succ (pred n) = n
   | _+1, _ => rfl
 
 theorem sub_one_add_one_eq_of_pos : ∀ {n}, 0 < n → (n - 1) + 1 = n
   | _+1, _ => rfl
 
+theorem eq_zero_or_eq_sub_one_add_one : ∀ {n}, n = 0 ∨ n = n - 1 + 1
+  | 0 => Or.inl rfl
+  | _+1 => Or.inr rfl
+
 @[simp] theorem pred_eq_sub_one : pred n = n - 1 := rfl
 
 /-! # sub theorems -/
@@ -806,6 +887,9 @@ theorem add_sub_of_le {a b : Nat} (h : a ≤ b) : a + (b - a) = b := by
     have : a ≤ b := Nat.le_of_succ_le h
     rw [sub_succ, Nat.succ_add, ← Nat.add_succ, Nat.succ_pred hne, ih this]
 
+theorem sub_one_cancel : ∀ {a b : Nat}, 0 < a → 0 < b → a - 1 = b - 1 → a = b
+  | _+1, _+1, _, _ => congrArg _
+
 @[simp] protected theorem sub_add_cancel {n m : Nat} (h : m ≤ n) : n - m + m = n := by
   rw [Nat.add_comm, Nat.add_sub_of_le h]
 
@@ -857,6 +941,17 @@ protected theorem sub_lt_sub_left : ∀ {k m n : Nat}, k < m → k < n → m - n
   | zero => rfl
   | succ n ih => simp only [ih, Nat.sub_succ]; decide
 
+protected theorem sub_lt_sub_right : ∀ {a b c : Nat}, c ≤ a → a < b → a - c < b - c
+  | 0, _, _, hle, h => by
+    rw [Nat.eq_zero_of_le_zero hle, Nat.sub_zero, Nat.sub_zero]
+    exact h
+  | _, _, 0, _, h => by
+    rw [Nat.sub_zero, Nat.sub_zero]
+    exact h
+  | _+1, _+1, _+1, hle, h => by
+    rw [Nat.add_sub_add_right, Nat.add_sub_add_right]
+    exact Nat.sub_lt_sub_right (le_of_succ_le_succ hle) (lt_of_succ_lt_succ h)
+
 protected theorem sub_self_add (n m : Nat) : n - (n + m) = 0 := by
   show (n + 0) - (n + m) = 0
   rw [Nat.add_sub_add_left, Nat.zero_sub]
@@ -935,6 +1030,9 @@ protected theorem sub_le_sub_right {n m : Nat} (h : n ≤ m) : ∀ k, n - k ≤
   | 0   => h
   | z+1 => pred_le_pred (Nat.sub_le_sub_right h z)
 
+protected theorem sub_le_add_right_sub (a i j : Nat) : a - i ≤ a + j - i :=
+  Nat.sub_le_sub_right (Nat.le_add_right ..) ..
+
 protected theorem lt_of_sub_ne_zero (h : n - m ≠ 0) : m < n :=
   Nat.not_le.1 (mt Nat.sub_eq_zero_of_le h)
 
@@ -947,6 +1045,9 @@ protected theorem lt_of_sub_pos (h : 0 < n - m) : m < n :=
 protected theorem lt_of_sub_eq_succ (h : m - n = succ l) : n < m :=
   Nat.lt_of_sub_pos (h ▸ Nat.zero_lt_succ _)
 
+protected theorem lt_of_sub_eq_sub_one (h : m - n = l + 1) : n < m :=
+  Nat.lt_of_sub_pos (h ▸ Nat.zero_lt_succ _)
+
 protected theorem sub_lt_left_of_lt_add {n k m : Nat} (H : n ≤ k) (h : k < n + m) : k - n < m := by
   have := Nat.sub_le_sub_right (succ_le_of_lt h) n
   rwa [Nat.add_sub_cancel_left, Nat.succ_sub H] at this
@@ -974,21 +1075,35 @@ protected theorem sub_eq_iff_eq_add {c : Nat} (h : b ≤ a) : a - b = c ↔ a =
 protected theorem sub_eq_iff_eq_add' {c : Nat} (h : b ≤ a) : a - b = c ↔ a = b + c := by
   rw [Nat.add_comm, Nat.sub_eq_iff_eq_add h]
 
-theorem mul_pred_left (n m : Nat) : pred n * m = n * m - m := by
+/-! ## Mul sub distrib -/
+
+theorem pred_mul (n m : Nat) : pred n * m = n * m - m := by
   cases n with
   | zero   => simp
   | succ n => rw [Nat.pred_succ, succ_mul, Nat.add_sub_cancel]
 
-/-! ## Mul sub distrib -/
+set_option linter.missingDocs false in
+@[deprecated (since := "2024-06-01")] abbrev mul_pred_left := @pred_mul
 
-theorem mul_pred_right (n m : Nat) : n * pred m = n * m - n := by
-  rw [Nat.mul_comm, mul_pred_left, Nat.mul_comm]
+protected theorem sub_one_mul  (n m : Nat) : (n - 1) * m = n * m - m := by
+  cases n with
+  | zero   => simp
+  | succ n =>
+    rw [Nat.add_sub_cancel, add_one_mul, Nat.add_sub_cancel]
 
+theorem mul_pred (n m : Nat) : n * pred m = n * m - n := by
+  rw [Nat.mul_comm, pred_mul, Nat.mul_comm]
+
+set_option linter.missingDocs false in
+@[deprecated (since := "2024-06-01")] abbrev mul_pred_right := @mul_pred
+
+theorem mul_sub_one (n m : Nat) : n * (m - 1) = n * m - n := by
+  rw [Nat.mul_comm, Nat.sub_one_mul , Nat.mul_comm]
 
 protected theorem mul_sub_right_distrib (n m k : Nat) : (n - m) * k = n * k - m * k := by
   induction m with
   | zero => simp
-  | succ m ih => rw [Nat.sub_succ, Nat.mul_pred_left, ih, succ_mul, Nat.sub_sub]; done
+  | succ m ih => rw [Nat.sub_succ, Nat.pred_mul, ih, succ_mul, Nat.sub_sub]; done
 
 protected theorem mul_sub_left_distrib (n m k : Nat) : n * (m - k) = n * m - n * k := by
   rw [Nat.mul_comm, Nat.mul_sub_right_distrib, Nat.mul_comm m n, Nat.mul_comm n k]

src/Init/Data/Nat/Bitwise/Lemmas.lean

@@ -90,6 +90,10 @@ noncomputable def div2Induction {motive : Nat → Sort u}
   unfold testBit
   simp [shiftRight_succ_inside]
 
+@[simp] theorem testBit_add_one (x i : Nat) : testBit x (i + 1) = testBit (x/2) i := by
+  unfold testBit
+  simp [shiftRight_succ_inside]
+
 theorem testBit_to_div_mod {x : Nat} : testBit x i = decide (x / 2^i % 2 = 1) := by
   induction i generalizing x with
   | zero =>

src/Init/Data/Nat/Gcd.lean

@@ -43,6 +43,9 @@ def gcd (m n : @& Nat) : Nat :=
 theorem gcd_succ (x y : Nat) : gcd (succ x) y = gcd (y % succ x) (succ x) := by
   rw [gcd]; rfl
 
+theorem gcd_add_one (x y : Nat) : gcd (x + 1) y = gcd (y % (x + 1)) (x + 1) := by
+  rw [gcd]; rfl
+
 @[simp] theorem gcd_one_left (n : Nat) : gcd 1 n = 1 := by
   rw [gcd_succ, mod_one]
   rfl

src/Init/Data/Nat/Lemmas.lean

@@ -101,6 +101,10 @@ protected theorem one_sub : ∀ n, 1 - n = if n = 0 then 1 else 0
 theorem succ_sub_sub_succ (n m k) : succ n - m - succ k = n - m - k := by
   rw [Nat.sub_sub, Nat.sub_sub, add_succ, succ_sub_succ]
 
+theorem add_sub_sub_add_right (n m k l : Nat) :
+    (n + l) - m - (k + l) = n - m - k := by
+  rw [Nat.sub_sub, Nat.sub_sub, ←Nat.add_assoc, Nat.add_sub_add_right]
+
 protected theorem sub_right_comm (m n k : Nat) : m - n - k = m - k - n := by
   rw [Nat.sub_sub, Nat.sub_sub, Nat.add_comm]
 
@@ -176,10 +180,12 @@ protected theorem sub_add_lt_sub (h₁ : m + k ≤ n) (h₂ : 0 < k) : n - (m +
   rw [← Nat.sub_sub]; exact Nat.sub_lt_of_pos_le h₂ (Nat.le_sub_of_add_le' h₁)
 
 theorem sub_one_lt_of_le (h₀ : 0 < a) (h₁ : a ≤ b) : a - 1 < b :=
-  Nat.lt_of_lt_of_le (Nat.pred_lt' h₀) h₁
+  Nat.lt_of_lt_of_le (Nat.pred_lt_of_lt h₀) h₁
 
 theorem sub_lt_succ (a b) : a - b < succ a := lt_succ_of_le (sub_le a b)
 
+theorem sub_lt_add_one (a b) : a - b < a + 1 := lt_add_one_of_le (sub_le a b)
+
 theorem sub_one_sub_lt (h : i < n) : n - 1 - i < n := by
   rw [Nat.sub_right_comm]; exact Nat.sub_one_lt_of_le (Nat.sub_pos_of_lt h) (Nat.sub_le ..)
 
@@ -206,13 +212,19 @@ instance : Std.IdempotentOp (α := Nat) min := ⟨Nat.min_self⟩
 
 @[simp] protected theorem min_zero (a) : min a 0 = 0 := Nat.min_eq_right (Nat.zero_le _)
 
-protected theorem min_assoc : ∀ (a b c : Nat), min (min a b) c = min a (min b c)
+@[simp] protected theorem min_assoc : ∀ (a b c : Nat), min (min a b) c = min a (min b c)
   | 0, _, _ => by rw [Nat.zero_min, Nat.zero_min, Nat.zero_min]
   | _, 0, _ => by rw [Nat.zero_min, Nat.min_zero, Nat.zero_min]
   | _, _, 0 => by rw [Nat.min_zero, Nat.min_zero, Nat.min_zero]
   | _+1, _+1, _+1 => by simp only [Nat.succ_min_succ]; exact congrArg succ <| Nat.min_assoc ..
 instance : Std.Associative (α := Nat) min := ⟨Nat.min_assoc⟩
 
+@[simp] protected theorem min_self_assoc {m n : Nat} : min m (min m n) = min m n := by
+  rw [← Nat.min_assoc, Nat.min_self]
+
+@[simp] protected theorem min_self_assoc' {m n : Nat} : min n (min m n) = min n m := by
+  rw [Nat.min_comm m n, ← Nat.min_assoc, Nat.min_self]
+
 protected theorem sub_sub_eq_min : ∀ (a b : Nat), a - (a - b) = min a b
   | 0, _ => by rw [Nat.zero_sub, Nat.zero_min]
   | _, 0 => by rw [Nat.sub_zero, Nat.sub_self, Nat.min_zero]
@@ -479,6 +491,9 @@ protected theorem mul_lt_mul_of_lt_of_lt {a b c d : Nat} (hac : a < c) (hbd : b
 theorem succ_mul_succ (a b) : succ a * succ b = a * b + a + b + 1 := by
   rw [succ_mul, mul_succ]; rfl
 
+theorem add_one_mul_add_one (a b : Nat) : (a + 1) * (b + 1) = a * b + a + b + 1 := by
+  rw [add_one_mul, mul_add_one]; rfl
+
 theorem mul_le_add_right (m k n : Nat) : k * m ≤ m + n ↔ (k-1) * m ≤ n := by
   match k with
   | 0 =>
@@ -562,6 +577,9 @@ theorem add_mod (a b n : Nat) : (a + b) % n = ((a % n) + (b % n)) % n := by
 theorem pow_succ' {m n : Nat} : m ^ n.succ = m * m ^ n := by
   rw [Nat.pow_succ, Nat.mul_comm]
 
+theorem pow_add_one' {m n : Nat} : m ^ (n + 1) = m * m ^ n := by
+  rw [Nat.pow_add_one, Nat.mul_comm]
+
 @[simp] theorem pow_eq {m n : Nat} : m.pow n = m ^ n := rfl
 
 theorem one_shiftLeft (n : Nat) : 1 <<< n = 2 ^ n := by rw [shiftLeft_eq, Nat.one_mul]
@@ -790,6 +808,11 @@ theorem shiftRight_succ_inside : ∀m n, m >>> (n+1) = (m/2) >>> n
   | 0 => by simp [shiftRight]
   | n + 1 => by simp [shiftRight, zero_shiftRight n, shiftRight_succ]
 
+theorem shiftLeft_add (m n : Nat) : ∀ k, m <<< (n + k) = (m <<< n) <<< k
+  | 0 => rfl
+  | k + 1 => by simp [← Nat.add_assoc, shiftLeft_add _ _ k, shiftLeft_succ]
+
+@[deprecated shiftLeft_add (since := "2024-06-02")]
 theorem shiftLeft_shiftLeft (m n : Nat) : ∀ k, (m <<< n) <<< k = m <<< (n + k)
   | 0 => rfl
   | k + 1 => by simp [← Nat.add_assoc, shiftLeft_shiftLeft _ _ k, shiftLeft_succ]

src/Init/Data/Option/Instances.lean

@@ -26,7 +26,7 @@ instance : Membership α (Option α) := ⟨fun a b => b = some a⟩
 instance [DecidableEq α] (j : α) (o : Option α) : Decidable (j ∈ o) :=
   inferInstanceAs <| Decidable (o = some j)
 
-theorem isNone_iff_eq_none {o : Option α} : o.isNone ↔ o = none :=
+@[simp] theorem isNone_iff_eq_none {o : Option α} : o.isNone ↔ o = none :=
   ⟨Option.eq_none_of_isNone, fun e => e.symm ▸ rfl⟩
 
 theorem some_inj {a b : α} : some a = some b ↔ a = b := by simp; rfl
@@ -72,7 +72,7 @@ satisfy `p`, using the proof to apply `f`.
 
 /-- Map a monadic function which returns `Unit` over an `Option`. -/
 @[inline] protected def forM [Pure m] : Option α → (α → m PUnit) → m PUnit
-  | none  , _ => pure ()
+  | none  , _ => pure ⟨⟩
   | some a, f => f a
 
 instance : ForM m (Option α) α :=

src/Init/Data/String/Basic.lean

@@ -1,12 +1,13 @@
 /-
 Copyright (c) 2016 Microsoft Corporation. All rights reserved.
 Released under Apache 2.0 license as described in the file LICENSE.
-Author: Leonardo de Moura
+Author: Leonardo de Moura, Mario Carneiro
 -/
 prelude
 import Init.Data.List.Basic
 import Init.Data.Char.Basic
 import Init.Data.Option.Basic
+
 universe u
 
 def List.asString (s : List Char) : String :=
@@ -256,7 +257,26 @@ def atEnd : (@& String) → (@& Pos) → Bool
   | s, p => p.byteIdx ≥ utf8ByteSize s
 
 /--
-Similar to `get` but runtime does not perform bounds check.
+Returns the character at position `p` of a string.
+If `p` is not a valid position, returns `(default : Char)`.
+
+Requires evidence, `h`, that `p` is within bounds
+instead of performing a runtime bounds check as in `get`.
+
+Examples:
+* `"abc".get' 0 (by decide) = 'a'`
+* `let lean := "L∃∀N"; lean.get' (0 |> lean.next |> lean.next) (by decide) = '∀'`
+
+A typical pattern combines `get'` with a dependent if-else expression
+to avoid the overhead of an additional bounds check. For example:
+```
+def getInBounds? (s : String) (p : String.Pos) : Option Char :=
+  if h : s.atEnd p then none else some (s.get' p h)
+```
+
+Even with evidence of `¬ s.atEnd p`,
+`p` may be invalid if a byte index points into the middle of a multi-byte UTF-8 character.
+For example, `"L∃∀N".get' ⟨2⟩ (by decide) = (default : Char)`.
 -/
 @[extern "lean_string_utf8_get_fast"]
 def get' (s : @& String) (p : @& Pos) (h : ¬ s.atEnd p) : Char :=
@@ -264,22 +284,41 @@ def get' (s : @& String) (p : @& Pos) (h : ¬ s.atEnd p) : Char :=
   | ⟨s⟩ => utf8GetAux s 0 p
 
 /--
-Similar to `next` but runtime does not perform bounds check.
+Returns the next position in a string after position `p`.
+If `p` is not a valid position, the result is unspecified.
+
+Requires evidence, `h`, that `p` is within bounds
+instead of performing a runtime bounds check as in `next`.
+
+Examples:
+* `let abc := "abc"; abc.get (abc.next' 0 (by decide)) = 'b'`
+
+A typical pattern combines `next'` with a dependent if-else expression
+to avoid the overhead of an additional bounds check. For example:
+```
+def next? (s: String) (p : String.Pos) : Option Char :=
+  if h : s.atEnd p then none else s.get (s.next' p h)
+```
 -/
 @[extern "lean_string_utf8_next_fast"]
 def next' (s : @& String) (p : @& Pos) (h : ¬ s.atEnd p) : Pos :=
   let c := get s p
   p + c
 
-theorem one_le_csize (c : Char) : 1 ≤ csize c := by
-  repeat first | apply iteInduction (motive := (1 ≤ UInt32.toNat ·)) <;> intros | decide
+theorem _root_.Char.utf8Size_pos (c : Char) : 0 < c.utf8Size := by
+  repeat first | apply iteInduction (motive := (0 < ·)) <;> intros | decide
+
+theorem _root_.Char.utf8Size_le_four (c : Char) : c.utf8Size ≤ 4 := by
+  repeat first | apply iteInduction (motive := (· ≤ 4)) <;> intros | decide
+
+@[deprecated Char.utf8Size_pos (since := "2026-06-04")] abbrev one_le_csize := Char.utf8Size_pos
 
 @[simp] theorem pos_lt_eq (p₁ p₂ : Pos) : (p₁ < p₂) = (p₁.1 < p₂.1) := rfl
 
-@[simp] theorem pos_add_char (p : Pos) (c : Char) : (p + c).byteIdx = p.byteIdx + csize c := rfl
+@[simp] theorem pos_add_char (p : Pos) (c : Char) : (p + c).byteIdx = p.byteIdx + c.utf8Size := rfl
 
 theorem lt_next (s : String) (i : Pos) : i.1 < (s.next i).1 :=
-  Nat.add_lt_add_left (one_le_csize _) _
+  Nat.add_lt_add_left (Char.utf8Size_pos _) _
 
 theorem utf8PrevAux_lt_of_pos : ∀ (cs : List Char) (i p : Pos), p ≠ 0 →
     (utf8PrevAux cs i p).1 < p.1
@@ -289,7 +328,7 @@ theorem utf8PrevAux_lt_of_pos : ∀ (cs : List Char) (i p : Pos), p ≠ 0 →
   | c::cs, i, p, h => by
     simp [utf8PrevAux]
     apply iteInduction (motive := (Pos.byteIdx · < _)) <;> intro h'
-    next => exact h' ▸ Nat.add_lt_add_left (one_le_csize _) _
+    next => exact h' ▸ Nat.add_lt_add_left (Char.utf8Size_pos _) _
     next => exact utf8PrevAux_lt_of_pos _ _ _ h
 
 theorem prev_lt_of_pos (s : String) (i : Pos) (h : i ≠ 0) : (s.prev i).1 < i.1 := by
@@ -305,6 +344,15 @@ def posOfAux (s : String) (c : Char) (stopPos : Pos) (pos : Pos) : Pos :=
   else pos
 termination_by stopPos.1 - pos.1
 
+/--
+Returns the position of the first occurrence of a character, `c`, in `s`.
+If `s` does not contain `c`, returns `s.endPos`.
+
+Examples:
+* `"abba".posOf 'a' = ⟨0⟩`
+* `"abba".posOf 'z' = ⟨4⟩`
+* `"L∃∀N".posOf '∀' = ⟨4⟩`
+-/
 @[inline] def posOf (s : String) (c : Char) : Pos :=
   posOfAux s c s.endPos 0
 
@@ -317,6 +365,15 @@ def revPosOfAux (s : String) (c : Char) (pos : Pos) : Option Pos :=
     else revPosOfAux s c pos
 termination_by pos.1
 
+/--
+Returns the position of the last occurrence of a character, `c`, in `s`.
+If `s` does not contain `c`, returns `none`.
+
+Examples:
+* `"abba".posOf 'a' = some ⟨3⟩`
+* `"abba".posOf 'z' = none`
+* `"L∃∀N".posOf '∀' = some ⟨4⟩`
+-/
 def revPosOf (s : String) (c : Char) : Option Pos :=
   revPosOfAux s c s.endPos
 
@@ -424,7 +481,7 @@ decreasing_by
   focus
     rename_i i₀ j₀ _ eq h'
     rw [show (s.next i₀ - sep.next j₀).1 = (i₀ - j₀).1 by
-      show (_ + csize _) - (_ + csize _) = _
+      show (_ + Char.utf8Size _) - (_ + Char.utf8Size _) = _
       rw [(beq_iff_eq ..).1 eq, Nat.add_sub_add_right]; rfl]
     right; exact Nat.sub_lt_sub_left
       (Nat.lt_of_le_of_lt (Nat.le_add_right ..) (Nat.gt_of_not_le (mt decide_eq_true h')))
@@ -455,6 +512,7 @@ instance : Inhabited String := ⟨""⟩
 
 instance : Append String := ⟨String.append⟩
 
+@[deprecated push (since := "2024-04-06")]
 def str : String → Char → String := push
 
 def pushn (s : String) (c : Char) (n : Nat) : String :=
@@ -671,18 +729,18 @@ theorem set_next_add (s : String) (i : Pos) (c : Char) (b₁ b₂)
   simp [next, get, set, endPos, utf8ByteSize] at h ⊢
   rw [Nat.add_comm i.1, Nat.add_assoc] at h ⊢
   let rec foo : ∀ cs a b₁ b₂,
-    csize (utf8GetAux cs a i) + b₁ = utf8ByteSize.go cs + b₂ →
-    csize (utf8GetAux (utf8SetAux c cs a i) a i) + b₁ = utf8ByteSize.go (utf8SetAux c cs a i) + b₂
+    (utf8GetAux cs a i).utf8Size + b₁ = utf8ByteSize.go cs + b₂ →
+    (utf8GetAux (utf8SetAux c cs a i) a i).utf8Size + b₁ = utf8ByteSize.go (utf8SetAux c cs a i) + b₂
   | [], _, _, _, h => h
   | c'::cs, a, b₁, b₂, h => by
     unfold utf8SetAux
-    apply iteInduction (motive := fun p => csize (utf8GetAux p a i) + b₁ = utf8ByteSize.go p + b₂) <;>
+    apply iteInduction (motive := fun p => (utf8GetAux p a i).utf8Size + b₁ = utf8ByteSize.go p + b₂) <;>
       intro h' <;> simp [utf8GetAux, h', utf8ByteSize.go] at h ⊢
     next =>
       rw [Nat.add_assoc, Nat.add_left_comm] at h ⊢; rw [Nat.add_left_cancel h]
     next =>
       rw [Nat.add_assoc] at h ⊢
-      refine foo cs (a + c') b₁ (csize c' + b₂) h
+      refine foo cs (a + c') b₁ (c'.utf8Size + b₂) h
   exact foo s.1 0 _ _ h
 
 theorem mapAux_lemma (s : String) (i : Pos) (c : Char) (h : ¬s.atEnd i) :
@@ -735,7 +793,7 @@ where
     else true
   termination_by stop1.1 - off1.1
   decreasing_by
-    have := Nat.sub_lt_sub_left _h (Nat.add_lt_add_left (one_le_csize c₁) off1.1)
+    have := Nat.sub_lt_sub_left _h (Nat.add_lt_add_left c₁.utf8Size_pos off1.1)
     decreasing_tactic
 
 /-- Return true iff `p` is a prefix of `s` -/
@@ -1018,5 +1076,145 @@ def decapitalize (s : String) :=
 
 end String
 
-protected def Char.toString (c : Char) : String :=
+namespace Char
+
+protected def toString (c : Char) : String :=
   String.singleton c
+
+@[simp] theorem length_toString (c : Char) : c.toString.length = 1 := rfl
+
+end Char
+
+namespace String
+
+theorem ext {s₁ s₂ : String} (h : s₁.data = s₂.data) : s₁ = s₂ :=
+  show ⟨s₁.data⟩ = (⟨s₂.data⟩ : String) from h ▸ rfl
+
+theorem ext_iff {s₁ s₂ : String} : s₁ = s₂ ↔ s₁.data = s₂.data := ⟨fun h => h ▸ rfl, ext⟩
+
+@[simp] theorem default_eq : default = "" := rfl
+
+@[simp] theorem length_mk (s : List Char) : (String.mk s).length = s.length := rfl
+
+@[simp] theorem length_empty : "".length = 0 := rfl
+
+@[simp] theorem length_singleton (c : Char) : (String.singleton c).length = 1 := rfl
+
+@[simp] theorem length_push (c : Char) : (String.push s c).length = s.length + 1 := by
+  rw [push, length_mk, List.length_append, List.length_singleton, Nat.succ.injEq]
+  rfl
+
+@[simp] theorem length_pushn (c : Char) (n : Nat) : (pushn s c n).length = s.length + n := by
+  unfold pushn; induction n <;> simp [Nat.repeat, Nat.add_assoc, *]
+
+@[simp] theorem length_append (s t : String) : (s ++ t).length = s.length + t.length := by
+  simp only [length, append, List.length_append]
+
+@[simp] theorem data_push (s : String) (c : Char) : (s.push c).data = s.data ++ [c] := rfl
+
+@[simp] theorem data_append (s t : String) : (s ++ t).data = s.data ++ t.data := rfl
+
+attribute [simp] toList -- prefer `String.data` over `String.toList` in lemmas
+
+theorem lt_iff (s t : String) : s < t ↔ s.data < t.data := .rfl
+
+namespace Pos
+
+@[simp] theorem byteIdx_zero : (0 : Pos).byteIdx = 0 := rfl
+
+theorem byteIdx_mk (n : Nat) : byteIdx ⟨n⟩ = n := rfl
+
+@[simp] theorem mk_zero : ⟨0⟩ = (0 : Pos) := rfl
+
+@[simp] theorem mk_byteIdx (p : Pos) : ⟨p.byteIdx⟩ = p := rfl
+
+theorem ext {i₁ i₂ : Pos} (h : i₁.byteIdx = i₂.byteIdx) : i₁ = i₂ :=
+  show ⟨i₁.byteIdx⟩ = (⟨i₂.byteIdx⟩ : Pos) from h ▸ rfl
+
+theorem ext_iff {i₁ i₂ : Pos} : i₁ = i₂ ↔ i₁.byteIdx = i₂.byteIdx := ⟨fun h => h ▸ rfl, ext⟩
+
+@[simp] theorem add_byteIdx (p₁ p₂ : Pos) : (p₁ + p₂).byteIdx = p₁.byteIdx + p₂.byteIdx := rfl
+
+theorem add_eq (p₁ p₂ : Pos) : p₁ + p₂ = ⟨p₁.byteIdx + p₂.byteIdx⟩ := rfl
+
+@[simp] theorem sub_byteIdx (p₁ p₂ : Pos) : (p₁ - p₂).byteIdx = p₁.byteIdx - p₂.byteIdx := rfl
+
+theorem sub_eq (p₁ p₂ : Pos) : p₁ - p₂ = ⟨p₁.byteIdx - p₂.byteIdx⟩ := rfl
+
+@[simp] theorem addChar_byteIdx (p : Pos) (c : Char) : (p + c).byteIdx = p.byteIdx + c.utf8Size := rfl
+
+theorem addChar_eq (p : Pos) (c : Char) : p + c = ⟨p.byteIdx + c.utf8Size⟩ := rfl
+
+theorem zero_addChar_byteIdx (c : Char) : ((0 : Pos) + c).byteIdx = c.utf8Size := by
+  simp only [addChar_byteIdx, byteIdx_zero, Nat.zero_add]
+
+theorem zero_addChar_eq (c : Char) : (0 : Pos) + c = ⟨c.utf8Size⟩ := by rw [← zero_addChar_byteIdx]
+
+theorem addChar_right_comm (p : Pos) (c₁ c₂ : Char) : p + c₁ + c₂ = p + c₂ + c₁ := by
+  apply ext
+  repeat rw [pos_add_char]
+  apply Nat.add_right_comm
+
+theorem ne_of_lt {i₁ i₂ : Pos} (h : i₁ < i₂) : i₁ ≠ i₂ := mt ext_iff.1 (Nat.ne_of_lt h)
+
+theorem ne_of_gt {i₁ i₂ : Pos} (h : i₁ < i₂) : i₂ ≠ i₁ := (ne_of_lt h).symm
+
+@[simp] theorem addString_byteIdx (p : Pos) (s : String) :
+    (p + s).byteIdx = p.byteIdx + s.utf8ByteSize := rfl
+
+theorem addString_eq (p : Pos) (s : String) : p + s = ⟨p.byteIdx + s.utf8ByteSize⟩ := rfl
+
+theorem zero_addString_byteIdx (s : String) : ((0 : Pos) + s).byteIdx = s.utf8ByteSize := by
+  simp only [addString_byteIdx, byteIdx_zero, Nat.zero_add]
+
+theorem zero_addString_eq (s : String) : (0 : Pos) + s = ⟨s.utf8ByteSize⟩ := by
+  rw [← zero_addString_byteIdx]
+
+theorem le_iff {i₁ i₂ : Pos} : i₁ ≤ i₂ ↔ i₁.byteIdx ≤ i₂.byteIdx := .rfl
+
+@[simp] theorem mk_le_mk {i₁ i₂ : Nat} : Pos.mk i₁ ≤ Pos.mk i₂ ↔ i₁ ≤ i₂ := .rfl
+
+theorem lt_iff {i₁ i₂ : Pos} : i₁ < i₂ ↔ i₁.byteIdx < i₂.byteIdx := .rfl
+
+@[simp] theorem mk_lt_mk {i₁ i₂ : Nat} : Pos.mk i₁ < Pos.mk i₂ ↔ i₁ < i₂ := .rfl
+
+end Pos
+
+@[simp] theorem get!_eq_get (s : String) (p : Pos) : get! s p = get s p := rfl
+
+theorem lt_next' (s : String) (p : Pos) : p < next s p := lt_next ..
+
+@[simp] theorem prev_zero (s : String) : prev s 0 = 0 := rfl
+
+@[simp] theorem get'_eq (s : String) (p : Pos) (h) : get' s p h = get s p := rfl
+
+@[simp] theorem next'_eq (s : String) (p : Pos) (h) : next' s p h = next s p := rfl
+
+-- `toSubstring'` is just a synonym for `toSubstring` without the `@[inline]` attribute
+-- so for proving can be unfolded.
+attribute [simp] toSubstring'
+
+theorem singleton_eq (c : Char) : singleton c = ⟨[c]⟩ := rfl
+
+@[simp] theorem data_singleton (c : Char) : (singleton c).data = [c] := rfl
+
+@[simp] theorem append_empty (s : String) : s ++ "" = s := ext (List.append_nil _)
+
+@[simp] theorem empty_append (s : String) : "" ++ s = s := rfl
+
+theorem append_assoc (s₁ s₂ s₃ : String) : (s₁ ++ s₂) ++ s₃ = s₁ ++ (s₂ ++ s₃) :=
+  ext (List.append_assoc ..)
+
+end String
+
+open String
+
+namespace Substring
+
+@[simp] theorem prev_zero (s : Substring) : s.prev 0 = 0 := by simp [prev, Pos.add_eq, Pos.byteIdx_zero]
+
+@[simp] theorem prevn_zero (s : Substring) : ∀ n, s.prevn n 0 = 0
+  | 0 => rfl
+  | n+1 => by simp [prevn, prevn_zero s n]
+
+end Substring

src/Init/Data/String/Extra.lean

@@ -63,10 +63,10 @@ where
   loop (i : Nat) : Option Unit := do
     if i < a.size then
       let c ← utf8DecodeChar? a i
-      loop (i + csize c)
+      loop (i + c.utf8Size)
     else pure ()
   termination_by a.size - i
-  decreasing_by exact Nat.sub_lt_sub_left ‹_› (Nat.lt_add_of_pos_right (one_le_csize c))
+  decreasing_by exact Nat.sub_lt_sub_left ‹_› (Nat.lt_add_of_pos_right c.utf8Size_pos)
 
 /-- Converts a [UTF-8](https://en.wikipedia.org/wiki/UTF-8) encoded `ByteArray` string to `String`. -/
 @[extern "lean_string_from_utf8"]
@@ -76,10 +76,10 @@ where
   loop (i : Nat) (acc : String) : String :=
     if i < a.size then
       let c := (utf8DecodeChar? a i).getD default
-      loop (i + csize c) (acc.push c)
+      loop (i + c.utf8Size) (acc.push c)
     else acc
   termination_by a.size - i
-  decreasing_by exact Nat.sub_lt_sub_left ‹_› (Nat.lt_add_of_pos_right (one_le_csize c))
+  decreasing_by exact Nat.sub_lt_sub_left ‹_› (Nat.lt_add_of_pos_right c.utf8Size_pos)
 
 /-- Converts a [UTF-8](https://en.wikipedia.org/wiki/UTF-8) encoded `ByteArray` string to `String`,
 or returns `none` if `a` is not properly UTF-8 encoded. -/
@@ -108,8 +108,8 @@ def utf8EncodeChar (c : Char) : List UInt8 :=
      (v >>>  6).toUInt8 &&& 0x3f ||| 0x80,
               v.toUInt8 &&& 0x3f ||| 0x80]
 
-@[simp] theorem length_utf8EncodeChar (c : Char) : (utf8EncodeChar c).length = csize c := by
-  simp [csize, utf8EncodeChar, Char.utf8Size]
+@[simp] theorem length_utf8EncodeChar (c : Char) : (utf8EncodeChar c).length = c.utf8Size := by
+  simp [Char.utf8Size, utf8EncodeChar]
   cases Decidable.em (c.val ≤ 0x7f) <;> simp [*]
   cases Decidable.em (c.val ≤ 0x7ff) <;> simp [*]
   cases Decidable.em (c.val ≤ 0xffff) <;> simp [*]
@@ -221,11 +221,11 @@ where
   termination_by text.utf8ByteSize - pos.byteIdx
   decreasing_by
     decreasing_with
-      show text.utf8ByteSize - (text.next' (text.next' pos _) _).byteIdx < text.utf8ByteSize - pos.byteIdx
+      show text.utf8ByteSize - (text.next (text.next pos)).byteIdx < text.utf8ByteSize - pos.byteIdx
       have k := Nat.gt_of_not_le <| mt decide_eq_true h
       exact Nat.sub_lt_sub_left k (Nat.lt_trans (String.lt_next text pos) (String.lt_next _ _))
     decreasing_with
-      show text.utf8ByteSize - (text.next' pos _).byteIdx < text.utf8ByteSize - pos.byteIdx
+      show text.utf8ByteSize - (text.next pos).byteIdx < text.utf8ByteSize - pos.byteIdx
       have k := Nat.gt_of_not_le <| mt decide_eq_true h
       exact Nat.sub_lt_sub_left k (String.lt_next _ _)
 

src/Init/GetElem.lean

@@ -141,12 +141,16 @@ instance : GetElem (List α) Nat α fun as i => i < as.length where
 
 instance : LawfulGetElem (List α) Nat α fun as i => i < as.length where
 
-@[simp] theorem cons_getElem_zero (a : α) (as : List α) (h : 0 < (a :: as).length) : getElem (a :: as) 0 h = a := by
+@[simp] theorem getElem_cons_zero (a : α) (as : List α) (h : 0 < (a :: as).length) : getElem (a :: as) 0 h = a := by
   rfl
 
-@[simp] theorem cons_getElem_succ (a : α) (as : List α) (i : Nat) (h : i + 1 < (a :: as).length) : getElem (a :: as) (i+1) h = getElem as i (Nat.lt_of_succ_lt_succ h) := by
+@[deprecated (since := "2024-6-12")] abbrev cons_getElem_zero := @getElem_cons_zero
+
+@[simp] theorem getElem_cons_succ (a : α) (as : List α) (i : Nat) (h : i + 1 < (a :: as).length) : getElem (a :: as) (i+1) h = getElem as i (Nat.lt_of_succ_lt_succ h) := by
   rfl
 
+@[deprecated (since := "2024-6-12")] abbrev cons_getElem_succ := @getElem_cons_succ
+
 theorem get_drop_eq_drop (as : List α) (i : Nat) (h : i < as.length) : as[i] :: as.drop (i+1) = as.drop i :=
   match as, i with
   | _::_, 0   => rfl

src/Init/Notation.lean

@@ -558,6 +558,22 @@ syntax (name := runMeta) "run_meta " doSeq : command
 set_option linter.missingDocs false in
 syntax guardMsgsFilterSeverity := &"info" <|> &"warning" <|> &"error" <|> &"all"
 
+/--
+`#reduce <expression>` reduces the expression `<expression>` to its normal form. This
+involves applying reduction rules until no further reduction is possible.
+
+By default, proofs and types within the expression are not reduced. Use modifiers
+`(proofs := true)`  and `(types := true)` to reduce them.
+Recall that propositions are types in Lean.
+
+**Warning:** This can be a computationally expensive operation,
+especially for complex expressions.
+
+Consider using `#eval <expression>` for simple evaluation/execution
+of expressions.
+-/
+syntax (name := reduceCmd) "#reduce " (atomic("(" &"proofs" " := " &"true" ")"))? (atomic("(" &"types" " := " &"true" ")"))? term : command
+
 /--
 A message filter specification for `#guard_msgs`.
 - `info`, `warning`, `error`: capture messages with the given severity level.

src/Init/Omega/Int.lean

@@ -187,7 +187,7 @@ theorem ofNat_val_add {x y : Fin n} :
     (((x + y : Fin n)) : Int) = ((x : Int) + (y : Int)) % n := rfl
 
 theorem ofNat_val_sub {x y : Fin n} :
-    (((x - y : Fin n)) : Int) = ((x : Int) + ((n - y : Nat) : Int)) % n := rfl
+    (((x - y : Fin n)) : Int) = (((n - y : Nat) + (x : Int) : Int)) % n := rfl
 
 theorem ofNat_val_mul {x y : Fin n} :
     (((x * y : Fin n)) : Int) = ((x : Int) * (y : Int)) % n := rfl

src/Init/Omega/IntList.lean

@@ -28,8 +28,8 @@ def get (xs : IntList) (i : Nat) : Int := (xs.get? i).getD 0
 @[simp] theorem get_cons_succ : get (x :: xs) (i+1) = get xs i := rfl
 
 theorem get_map {xs : IntList} (h : f 0 = 0) : get (xs.map f) i = f (xs.get i) := by
-  simp only [get, List.get?_map]
-  cases xs.get? i <;> simp_all
+  simp only [get, List.get?_eq_getElem?, List.getElem?_map]
+  cases xs[i]? <;> simp_all
 
 theorem get_of_length_le {xs : IntList} (h : xs.length ≤ i) : xs.get i = 0 := by
   rw [get, List.get?_eq_none.mpr h]
@@ -66,8 +66,8 @@ theorem add_def (xs ys : IntList) :
   rfl
 
 @[simp] theorem add_get (xs ys : IntList) (i : Nat) : (xs + ys).get i = xs.get i + ys.get i := by
-  simp only [add_def, get, List.zipWithAll_get?, List.get?_eq_none]
-  cases xs.get? i <;> cases ys.get? i <;> simp
+  simp only [get, add_def, List.get?_eq_getElem?, List.getElem?_zipWithAll]
+  cases xs[i]? <;> cases ys[i]? <;> simp
 
 @[simp] theorem add_nil (xs : IntList) : xs + [] = xs := by simp [add_def]
 @[simp] theorem nil_add (xs : IntList) : [] + xs = xs := by simp [add_def]
@@ -83,8 +83,8 @@ theorem mul_def (xs ys : IntList) : xs * ys = List.zipWith (· * ·) xs ys :=
   rfl
 
 @[simp] theorem mul_get (xs ys : IntList) (i : Nat) : (xs * ys).get i = xs.get i * ys.get i := by
-  simp only [mul_def, get, List.zipWith_get?]
-  cases xs.get? i <;> cases ys.get? i <;> simp
+  simp only [get, mul_def, List.get?_eq_getElem?, List.getElem?_zipWith]
+  cases xs[i]? <;> cases ys[i]? <;> simp
 
 @[simp] theorem mul_nil_left : ([] : IntList) * ys = [] := rfl
 @[simp] theorem mul_nil_right : xs * ([] : IntList) = [] := List.zipWith_nil_right
@@ -98,8 +98,8 @@ instance : Neg IntList := ⟨neg⟩
 theorem neg_def (xs : IntList) : - xs = xs.map fun x => -x := rfl
 
 @[simp] theorem neg_get (xs : IntList) (i : Nat) : (- xs).get i = - xs.get i := by
-  simp only [neg_def, get, List.get?_map]
-  cases xs.get? i <;> simp
+  simp only [get, neg_def, List.get?_eq_getElem?, List.getElem?_map]
+  cases xs[i]? <;> simp
 
 @[simp] theorem neg_nil : (- ([] : IntList)) = [] := rfl
 @[simp] theorem neg_cons : (- (x::xs : IntList)) = -x :: -xs := rfl
@@ -124,8 +124,8 @@ instance : HMul Int IntList IntList where
 theorem smul_def (xs : IntList) (i : Int) : i * xs = xs.map fun x => i * x := rfl
 
 @[simp] theorem smul_get (xs : IntList) (a : Int) (i : Nat) : (a * xs).get i = a * xs.get i := by
-  simp only [smul_def, get, List.get?_map]
-  cases xs.get? i <;> simp
+  simp only [get, smul_def, List.get?_eq_getElem?, List.getElem?_map]
+  cases xs[i]? <;> simp
 
 @[simp] theorem smul_nil {i : Int} : i * ([] : IntList) = [] := rfl
 @[simp] theorem smul_cons {i : Int} : i * (x::xs : IntList) = i * x :: i * xs := rfl

src/Init/Prelude.lean

@@ -914,6 +914,9 @@ is `Bool` valued instead of `Prop` valued, and it also does not have any
 axioms like being reflexive or agreeing with `=`. It is mainly intended for
 programming applications. See `LawfulBEq` for a version that requires that
 `==` and `=` coincide.
+
+Typically we prefer to put the "more variable" term on the left,
+and the "more constant" term on the right.
 -/
 class BEq (α : Type u) where
   /-- Boolean equality, notated as `a == b`. -/
@@ -1068,11 +1071,15 @@ This type is special-cased by both the kernel and the compiler:
   library (usually [GMP](https://gmplib.org/)).
 -/
 inductive Nat where
-  /-- `Nat.zero`, normally written `0 : Nat`, is the smallest natural number.
-  This is one of the two constructors of `Nat`. -/
+  /-- `Nat.zero`, is the smallest natural number. This is one of the two
+  constructors of `Nat`. Using `Nat.zero` should usually be avoided in favor of
+  `0 : Nat` or simply `0`, in order to remain compatible with the simp normal
+  form defined by `Nat.zero_eq`. -/
   | zero : Nat
   /-- The successor function on natural numbers, `succ n = n + 1`.
-  This is one of the two constructors of `Nat`. -/
+  This is one of the two constructors of `Nat`. Using `succ n` should usually
+  be avoided in favor of `n + 1`, in order to remain compatible with the simp
+  normal form defined by `Nat.succ_eq_add_one`. -/
   | succ (n : Nat) : Nat
 
 instance : Inhabited Nat where
@@ -2196,15 +2203,11 @@ instance : DecidableEq Char :=
     | isFalse h => isFalse (Char.ne_of_val_ne h)
 
 /-- Returns the number of bytes required to encode this `Char` in UTF-8. -/
-def Char.utf8Size (c : Char) : UInt32 :=
+def Char.utf8Size (c : Char) : Nat :=
   let v := c.val
-  ite (LE.le v (UInt32.ofNatCore 0x7F (by decide)))
-    (UInt32.ofNatCore 1 (by decide))
-    (ite (LE.le v (UInt32.ofNatCore 0x7FF (by decide)))
-      (UInt32.ofNatCore 2 (by decide))
-      (ite (LE.le v (UInt32.ofNatCore 0xFFFF (by decide)))
-        (UInt32.ofNatCore 3 (by decide))
-        (UInt32.ofNatCore 4 (by decide))))
+  ite (LE.le v (UInt32.ofNatCore 0x7F (by decide))) 1
+    (ite (LE.le v (UInt32.ofNatCore 0x7FF (by decide))) 2
+      (ite (LE.le v (UInt32.ofNatCore 0xFFFF (by decide))) 3 4))
 
 /--
 `Option α` is the type of values which are either `some a` for some `a : α`,
@@ -2303,24 +2306,6 @@ protected def List.hasDecEq {α : Type u} [DecidableEq α] : (a b : List α) →
 
 instance {α : Type u} [DecidableEq α] : DecidableEq (List α) := List.hasDecEq
 
-/--
-Folds a function over a list from the left:
-`foldl f z [a, b, c] = f (f (f z a) b) c`
--/
-@[specialize]
-def List.foldl {α : Type u} {β : Type v} (f : α → β → α) : (init : α) → List β → α
-  | a, nil      => a
-  | a, cons b l => foldl f (f a b) l
-
-/--
-`l.set n a` sets the value of list `l` at (zero-based) index `n` to `a`:
-`[a, b, c, d].set 1 b' = [a, b', c, d]`
--/
-def List.set : List α → Nat → α → List α
-  | cons _ as, 0,          b => cons b as
-  | cons a as, Nat.succ n, b => cons a (set as n b)
-  | nil,       _,          _ => nil
-
 /--
 The length of a list: `[].length = 0` and `(a :: l).length = l.length + 1`.
 
@@ -2347,11 +2332,6 @@ def List.lengthTR (as : List α) : Nat :=
 @[simp] theorem List.length_cons {α} (a : α) (as : List α) : Eq (cons a as).length as.length.succ :=
   rfl
 
-/-- `l.concat a` appends `a` at the *end* of `l`, that is, `l ++ [a]`. -/
-def List.concat {α : Type u} : List α → α → List α
-  | nil,       b => cons b nil
-  | cons a as, b => cons a (concat as b)
-
 /--
 `as.get i` returns the `i`'th element of the list `as`.
 This version of the function uses `i : Fin as.length` to ensure that it will
@@ -2361,6 +2341,29 @@ def List.get {α : Type u} : (as : List α) → Fin as.length → α
   | cons a _,  ⟨0, _⟩ => a
   | cons _ as, ⟨Nat.succ i, h⟩ => get as ⟨i, Nat.le_of_succ_le_succ h⟩
 
+/--
+`l.set n a` sets the value of list `l` at (zero-based) index `n` to `a`:
+`[a, b, c, d].set 1 b' = [a, b', c, d]`
+-/
+def List.set : List α → Nat → α → List α
+  | cons _ as, 0,          b => cons b as
+  | cons a as, Nat.succ n, b => cons a (set as n b)
+  | nil,       _,          _ => nil
+
+/--
+Folds a function over a list from the left:
+`foldl f z [a, b, c] = f (f (f z a) b) c`
+-/
+@[specialize]
+def List.foldl {α : Type u} {β : Type v} (f : α → β → α) : (init : α) → List β → α
+  | a, nil      => a
+  | a, cons b l => foldl f (f a b) l
+
+/-- `l.concat a` appends `a` at the *end* of `l`, that is, `l ++ [a]`. -/
+def List.concat {α : Type u} : List α → α → List α
+  | nil,       b => cons b nil
+  | cons a as, b => cons a (concat as b)
+
 /--
 `String` is the type of (UTF-8 encoded) strings.
 
@@ -2433,10 +2436,6 @@ instance : Inhabited Substring where
 @[inline] def Substring.bsize : Substring → Nat
   | ⟨_, b, e⟩ => e.byteIdx.sub b.byteIdx
 
-/-- Returns the number of bytes required to encode this `Char` in UTF-8. -/
-def String.csize (c : Char) : Nat :=
-  c.utf8Size.toNat
-
 /--
 The UTF-8 byte length of this string.
 This is overridden by the compiler to be cached and O(1).
@@ -2447,7 +2446,7 @@ def String.utf8ByteSize : (@& String) → Nat
 where
   go : List Char → Nat
    | .nil       => 0
-   | .cons c cs => hAdd (go cs) (csize c)
+   | .cons c cs => hAdd (go cs) c.utf8Size
 
 instance : HAdd String.Pos String.Pos String.Pos where
   hAdd p₁ p₂ := { byteIdx := hAdd p₁.byteIdx p₂.byteIdx }
@@ -2456,7 +2455,7 @@ instance : HSub String.Pos String.Pos String.Pos where
   hSub p₁ p₂ := { byteIdx := HSub.hSub p₁.byteIdx p₂.byteIdx }
 
 instance : HAdd String.Pos Char String.Pos where
-  hAdd p c := { byteIdx := hAdd p.byteIdx (String.csize c) }
+  hAdd p c := { byteIdx := hAdd p.byteIdx c.utf8Size }
 
 instance : HAdd String.Pos String String.Pos where
   hAdd p s := { byteIdx := hAdd p.byteIdx s.utf8ByteSize }

src/Init/System/IO.lean

@@ -253,12 +253,9 @@ instance : ToString TaskState := ⟨TaskState.toString⟩
 @[extern "lean_io_wait"] opaque wait (t : Task α) : BaseIO α :=
   return t.get
 
-local macro "nonempty_list" : tactic =>
-  `(tactic| exact Nat.zero_lt_succ _)
-
 /-- Wait until any of the tasks in the given list has finished, then return its result. -/
 @[extern "lean_io_wait_any"] opaque waitAny (tasks : @& List (Task α))
-    (h : tasks.length > 0 := by nonempty_list) : BaseIO α :=
+    (h : tasks.length > 0 := by exact Nat.zero_lt_succ _) : BaseIO α :=
   return tasks[0].get
 
 /-- Helper method for implementing "deterministic" timeouts. It is the number of "small" memory allocations performed by the current execution thread. -/

src/Init/Tactics.lean

@@ -699,7 +699,37 @@ The `have` tactic is for adding hypotheses to the local context of the main goal
   For example, given `h : p ∧ q ∧ r`, `have ⟨h₁, h₂, h₃⟩ := h` produces the
   hypotheses `h₁ : p`, `h₂ : q`, and `h₃ : r`.
 -/
-macro "have " d:haveDecl : tactic => `(tactic| refine_lift have $d:haveDecl; ?_)
+syntax "have " haveDecl : tactic
+macro_rules
+  -- special case: when given a nested `by` block, move it outside of the `refine` to enable
+  -- incrementality
+  | `(tactic| have%$haveTk $id:haveId $bs* : $type := by%$byTk $tacs*) => do
+    /-
+    We want to create the syntax
+    ```
+    focus
+      refine no_implicit_lambda% (have $id:haveId $bs* : $type := ?body; ?_)
+      case body => $tacs*
+    ```
+    However, we need to be very careful with the syntax infos involved:
+    * We want most infos up to `tacs` to be independent of changes inside it so that incrementality
+      is not prematurely disabled; we use the `have` and then the `by` token as the reference for
+      this. Note that if we did nothing, the reference would be the entire `have` input and so any
+      change to `tacs` would change every token synthesized below.
+    * For the single node of the `case` body, we *should not* change the ref as this makes sure the
+      entire tactic block is included in any "unsaved goals" message (which is emitted after
+      execution of all nested tactics so it is indeed safe for `evalCase` to ignore it for
+      incrementality).
+    * Even after setting the ref, we still need a `with_annotate_state` to show the correct tactic
+      state on `by` as the synthetic info derived from the ref is ignored for this purpose.
+    -/
+    let tac ← Lean.withRef byTk `(tactic| with_annotate_state $byTk ($tacs*))
+    let tac ← `(tacticSeq| $tac:tactic)
+    let tac ← Lean.withRef byTk `(tactic| case body => $(.mk tac):tacticSeq)
+    Lean.withRef haveTk `(tactic| focus
+      refine no_implicit_lambda% (have $id:haveId $bs* : $type := ?body; ?_)
+      $tac)
+  | `(tactic| have $d:haveDecl) => `(tactic| refine_lift have $d:haveDecl; ?_)
 
 /--
 Given a main goal `ctx ⊢ t`, `suffices h : t' from e` replaces the main goal with `ctx ⊢ t'`,
@@ -833,14 +863,41 @@ syntax (name := cases) "cases " casesTarget,+ (" using " term)? (inductionAlts)?
 syntax (name := renameI) "rename_i" (ppSpace colGt binderIdent)+ : tactic
 
 /--
-`repeat tac` repeatedly applies `tac` to the main goal until it fails.
-That is, if `tac` produces multiple subgoals, only subgoals up to the first failure will be visited.
-The `Batteries` library provides `repeat'` which repeats separately in each subgoal.
+`repeat tac` repeatedly applies `tac` so long as it succeeds.
+The tactic `tac` may be a tactic sequence, and if `tac` fails at any point in its execution,
+`repeat` will revert any partial changes that `tac` made to the tactic state.
+
+The tactic `tac` should eventually fail, otherwise `repeat tac` will run indefinitely.
+
+See also:
+* `try tac` is like `repeat tac` but will apply `tac` at most once.
+* `repeat' tac` recursively applies `tac` to each goal.
+* `first | tac1 | tac2` implements the backtracking used by `repeat`
 -/
 syntax "repeat " tacticSeq : tactic
 macro_rules
   | `(tactic| repeat $seq) => `(tactic| first | ($seq); repeat $seq | skip)
 
+/--
+`repeat' tac` recursively applies `tac` on all of the goals so long as it succeeds.
+That is to say, if `tac` produces multiple subgoals, then `repeat' tac` is applied to each of them.
+
+See also:
+* `repeat tac` simply repeatedly applies `tac`.
+* `repeat1' tac` is `repeat' tac` but requires that `tac` succeed for some goal at least once.
+-/
+syntax (name := repeat') "repeat' " tacticSeq : tactic
+
+/--
+`repeat1' tac` recursively applies to `tac` on all of the goals so long as it succeeds,
+but `repeat1' tac` fails if `tac` succeeds on none of the initial goals.
+
+See also:
+* `repeat tac` simply applies `tac` repeatedly.
+* `repeat' tac` is like `repeat1' tac` but it does not require that `tac` succeed at least once.
+-/
+syntax (name := repeat1') "repeat1' " tacticSeq : tactic
+
 /--
 `trivial` tries different simple tactics (e.g., `rfl`, `contradiction`, ...)
 to close the current goal.
@@ -1041,18 +1098,6 @@ This can be used to simulate the `specialize` and `apply at` tactics of Coq.
 -/
 syntax (name := replace) "replace" haveDecl : tactic
 
-/--
-`repeat' tac` runs `tac` on all of the goals to produce a new list of goals,
-then runs `tac` again on all of those goals, and repeats until `tac` fails on all remaining goals.
--/
-syntax (name := repeat') "repeat' " tacticSeq : tactic
-
-/--
-`repeat1' tac` applies `tac` to main goal at least once. If the application succeeds,
-the tactic is applied recursively to the generated subgoals until it eventually fails.
--/
-syntax (name := repeat1') "repeat1' " tacticSeq : tactic
-
 /-- `and_intros` applies `And.intro` until it does not make progress. -/
 syntax "and_intros" : tactic
 macro_rules | `(tactic| and_intros) => `(tactic| repeat' refine And.intro ?_ ?_)
@@ -1414,6 +1459,7 @@ have been simplified by using the modifier `↓`. Here is an example
 ```
 
 When multiple simp theorems are applicable, the simplifier uses the one with highest priority.
+The equational theorems of function are applied at very low priority (100 and below).
 If there are several with the same priority, it is uses the "most recent one". Example:
 ```lean
 @[simp high] theorem cond_true (a b : α) : cond true a b = a := rfl

src/Init/WFTactics.lean

@@ -32,7 +32,7 @@ before `omega` is available.
 -/
 syntax "decreasing_trivial_pre_omega" : tactic
 macro_rules | `(tactic| decreasing_trivial_pre_omega) => `(tactic| apply Nat.sub_succ_lt_self; assumption) -- a - (i+1) < a - i if i < a
-macro_rules | `(tactic| decreasing_trivial_pre_omega) => `(tactic| apply Nat.pred_lt'; assumption) -- i-1 < i if j < i
+macro_rules | `(tactic| decreasing_trivial_pre_omega) => `(tactic| apply Nat.pred_lt_of_lt; assumption) -- i-1 < i if j < i
 macro_rules | `(tactic| decreasing_trivial_pre_omega) => `(tactic| apply Nat.pred_lt; assumption)  -- i-1 < i if i ≠ 0
 
 

src/Lean/Compiler/IR/Basic.lean

@@ -24,12 +24,12 @@ abbrev Index := Nat
 /-- Variable identifier -/
 structure VarId where
   idx : Index
-  deriving Inhabited
+  deriving Inhabited, Repr
 
 /-- Join point identifier -/
 structure JoinPointId where
   idx : Index
-  deriving Inhabited
+  deriving Inhabited, Repr
 
 abbrev Index.lt (a b : Index) : Bool := a < b
 
@@ -83,7 +83,7 @@ inductive IRType where
   | irrelevant | object | tobject
   | struct (leanTypeName : Option Name) (types : Array IRType) : IRType
   | union (leanTypeName : Name) (types : Array IRType) : IRType
-  deriving Inhabited
+  deriving Inhabited, Repr
 
 namespace IRType
 
@@ -236,7 +236,7 @@ structure Param where
   x : VarId
   borrow : Bool
   ty : IRType
-  deriving Inhabited
+  deriving Inhabited, Repr
 
 @[export lean_ir_mk_param]
 def mkParam (x : VarId) (borrow : Bool) (ty : IRType) : Param := ⟨x, borrow, ty⟩

src/Lean/Compiler/IR/Borrow.lean

@@ -258,7 +258,8 @@ def preserveTailCall (x : VarId) (v : Expr) (b : FnBody) : M Unit := do
   let ctx ← read
   match v, b with
   | (Expr.fap g ys), (FnBody.ret (Arg.var z)) =>
-    if ctx.decls.any (·.name == g) && x == z then
+    -- NOTE: we currently support TCO for self-calls only
+    if ctx.currFn == g && x == z then
       let ps ← getParamInfo (ParamMap.Key.decl g)
       ownParamsUsingArgs ys ps
   | _, _ => pure ()

src/Lean/CoreM.lean

@@ -219,31 +219,31 @@ def saveState : CoreM SavedState := do
   return { toState := s, passedHearbeats := 0 }
 
 /--
-Incremental reuse primitive: if `reusableResult?` is `none`, runs `cont` with an action `save` that
-on execution returns the saved monadic state at this point including the heartbeats used by `cont`
-so far. If `reusableResult?` on the other hand is `some (a, state)`, restores full `state` including
-heartbeats used and returns `a`.
+Incremental reuse primitive: if `reusableResult?` is `none`, runs `act` and returns its result
+together with the saved monadic state after `act` including the heartbeats used by it. If
+`reusableResult?` on the other hand is `some (a, state)`, restores full `state` including heartbeats
+used and returns `(a, state)`.
 
 The intention is for steps that support incremental reuse to initially pass `none` as
-`reusableResult?` and call `save` as late as possible in `cont`. In a further run, if reuse is
-possible, `reusableResult?` should be set to the previous state and result, ensuring that the state
+`reusableResult?` and store the result and state in a snapshot. In a further run, if reuse is
+possible, `reusableResult?` should be set to the previous result and state, ensuring that the state
 after running `withRestoreOrSaveFull` is identical in both runs. Note however that necessarily this
-is only an approximation in the case of heartbeats as heartbeats used by `withRestoreOrSaveFull`, by
-the remainder of `cont` after calling `save`, as well as by reuse-handling code such as the one
-supplying `reusableResult?` are not accounted for.
+is only an approximation in the case of heartbeats as heartbeats used by `withRestoreOrSaveFull`
+itself after calling `act` as well as by reuse-handling code such as the one supplying
+`reusableResult?` are not accounted for.
 -/
 @[specialize] def withRestoreOrSaveFull (reusableResult? : Option (α × SavedState))
-    (cont : (save : CoreM SavedState) → CoreM α) : CoreM α := do
+    (act : CoreM α) : CoreM (α × SavedState) := do
   if let some (val, state) := reusableResult? then
     set state.toState
     IO.addHeartbeats state.passedHearbeats.toUInt64
-    return val
+    return (val, state)
 
   let startHeartbeats ← IO.getNumHeartbeats
-  cont (do
-    let s ← get
-    let stopHeartbeats ← IO.getNumHeartbeats
-    return { toState := s, passedHearbeats := stopHeartbeats - startHeartbeats })
+  let a ← act
+  let s ← get
+  let stopHeartbeats ← IO.getNumHeartbeats
+  return (a, { toState := s, passedHearbeats := stopHeartbeats - startHeartbeats })
 
 /-- Restore backtrackable parts of the state. -/
 def SavedState.restore (b : SavedState) : CoreM Unit :=

src/Lean/Data/HashMap.lean

@@ -15,6 +15,10 @@ def HashMapBucket.update {α : Type u} {β : Type v} (data : HashMapBucket α β
   ⟨ data.val.uset i d h,
     by erw [Array.size_set]; apply data.property ⟩
 
+@[simp] theorem HashMapBucket.size_update {α : Type u} {β : Type v} (data : HashMapBucket α β) (i : USize) (d : AssocList α β)
+    (h : i.toNat < data.val.size) : (data.update i d h).val.size = data.val.size := by
+  simp [update, Array.uset]
+
 structure HashMapImp (α : Type u) (β : Type v) where
   size       : Nat
   buckets    : HashMapBucket α β
@@ -108,7 +112,9 @@ def expand [Hashable α] (size : Nat) (buckets : HashMapBucket α β) : HashMapI
     let ⟨i, h⟩ := mkIdx (hash a) buckets.property
     let bkt    := buckets.val[i]
     if bkt.contains a then
-      (⟨size, buckets.update i (bkt.replace a b) h⟩, true)
+      -- make sure `bkt` is used linearly in the following call to `replace`
+      let buckets' := buckets.update i .nil h
+      (⟨size, buckets'.update i (bkt.replace a b) (by simpa [buckets'])⟩, true)
     else
       let size'    := size + 1
       let buckets' := buckets.update i (AssocList.cons a b bkt) h
@@ -139,7 +145,9 @@ def erase [BEq α] [Hashable α] (m : HashMapImp α β) (a : α) : HashMapImp α
     let ⟨i, h⟩ := mkIdx (hash a) buckets.property
     let bkt    := buckets.val[i]
     if bkt.contains a then
-      ⟨size - 1, buckets.update i (bkt.erase a) h⟩
+      -- make sure `bkt` is used linearly in the following call to `erase`
+      let buckets' := buckets.update i .nil h
+      ⟨size - 1, buckets'.update i (bkt.erase a) (by simpa [buckets'])⟩
     else
       ⟨size, buckets⟩
 

src/Lean/Data/HashSet.lean

@@ -16,6 +16,10 @@ def HashSetBucket.update {α : Type u} (data : HashSetBucket α) (i : USize) (d
   ⟨ data.val.uset i d h,
     by erw [Array.size_set]; apply data.property ⟩
 
+@[simp] theorem HashSetBucket.size_update {α : Type u} (data : HashSetBucket α) (i : USize) (d : List α) (h : i.toNat < data.val.size) :
+    (data.update i d h).val.size = data.val.size := by
+  simp [update, Array.uset]
+
 structure HashSetImp (α : Type u) where
   size       : Nat
   buckets    : HashSetBucket α
@@ -100,7 +104,10 @@ def insert [BEq α] [Hashable α] (m : HashSetImp α) (a : α) : HashSetImp α :
     let ⟨i, h⟩ := mkIdx (hash a) buckets.property
     let bkt    := buckets.val[i]
     if bkt.contains a
-    then ⟨size, buckets.update i (bkt.replace a a) h⟩
+    then
+      -- make sure `bkt` is used linearly in the following call to `replace`
+      let buckets' := buckets.update i .nil h
+      ⟨size, buckets'.update i (bkt.replace a a) (by simpa [buckets'])⟩
     else
       let size'    := size + 1
       let buckets' := buckets.update i (a :: bkt) h
@@ -114,7 +121,9 @@ def erase [BEq α] [Hashable α] (m : HashSetImp α) (a : α) : HashSetImp α :=
     let ⟨i, h⟩ := mkIdx (hash a) buckets.property
     let bkt    := buckets.val[i]
     if bkt.contains a then
-      ⟨size - 1, buckets.update i (bkt.erase a) h⟩
+      -- make sure `bkt` is used linearly in the following call to `erase`
+      let buckets' := buckets.update i .nil h
+      ⟨size - 1, buckets'.update i (bkt.erase a) (by simpa [buckets'])⟩
     else
       ⟨size, buckets⟩
 

src/Lean/Data/Lsp/LanguageFeatures.lean

@@ -333,8 +333,8 @@ def SemanticTokenType.names : Array String :=
     "event", "method", "macro", "modifier", "comment", "string", "number",
     "regexp", "operator", "decorator", "leanSorryLike"]
 
-def SemanticTokenType.toNat (type : SemanticTokenType) : Nat :=
-  type.toCtorIdx
+def SemanticTokenType.toNat (tokenType : SemanticTokenType) : Nat :=
+  tokenType.toCtorIdx
 
 -- sanity check
 -- TODO: restore after update-stage0

src/Lean/Data/Name.lean

@@ -120,6 +120,26 @@ def isInternalOrNum : Name → Bool
   | .num _ _ => true
   | _       => false
 
+/--
+Returns true if this a part of name that is internal or dynamically
+generated so that it may easily be changed.
+
+Generally, user code should not explicitly use internal names.
+-/
+def isInternalDetail : Name → Bool
+  | .str p s     =>
+    s.startsWith "_"
+      || matchPrefix s "eq_"
+      || matchPrefix s "match_"
+      || matchPrefix s "proof_"
+      || p.isInternalOrNum
+  | .num _ _     => true
+  | p            => p.isInternalOrNum
+where
+  /-- Check that a string begins with the given prefix, and then is only digit characters. -/
+  matchPrefix (s : String) (pre : String) :=
+    s.startsWith pre && (s |>.drop pre.length |>.all Char.isDigit)
+
 /--
 Checks whether the name is an implementation-detail hypothesis name.
 

src/Lean/Data/NameMap.lean

@@ -27,7 +27,7 @@ def insert (m : NameMap α) (n : Name) (a : α) := RBMap.insert m n a
 
 def contains (m : NameMap α) (n : Name) : Bool := RBMap.contains m n
 
-@[inline] def find? (m : NameMap α) (n : Name) : Option α := RBMap.find? m n
+def find? (m : NameMap α) (n : Name) : Option α := RBMap.find? m n
 
 instance : ForIn m (NameMap α) (Name × α) :=
   inferInstanceAs (ForIn _ (RBMap ..) ..)

src/Lean/Data/SMap.lean

@@ -74,6 +74,12 @@ def forM [Monad m] (s : SMap α β) (f : α → β → m PUnit) : m PUnit := do
   s.map₁.forM f
   s.map₂.forM f
 
+instance : ForM m (SMap α β) (α × β) where
+  forM s f := forM s fun x y => f (x, y)
+
+instance : ForIn m (SMap α β) (α × β) where
+  forIn := ForM.forIn
+
 /-- Move from stage 1 into stage 2. -/
 def switch (m : SMap α β) : SMap α β :=
   if m.stage₁ then { m with stage₁ := false } else m

src/Lean/Elab/App.lean

@@ -59,7 +59,7 @@ private def mkProjAndCheck (structName : Name) (idx : Nat) (e : Expr) : MetaM Ex
 def synthesizeAppInstMVars (instMVars : Array MVarId) (app : Expr) : TermElabM Unit :=
   for mvarId in instMVars do
     unless (← synthesizeInstMVarCore mvarId) do
-      registerSyntheticMVarWithCurrRef mvarId SyntheticMVarKind.typeClass
+      registerSyntheticMVarWithCurrRef mvarId (.typeClass none)
       registerMVarErrorImplicitArgInfo mvarId (← getRef) app
 
 /-- Return `some namedArg` if `namedArgs` contains an entry for `binderName`. -/
@@ -233,9 +233,7 @@ def eraseNamedArg (binderName : Name) : M Unit :=
 private def addNewArg (argName : Name) (arg : Expr) : M Unit := do
   modify fun s => { s with f := mkApp s.f arg, fType := s.fType.bindingBody!.instantiate1 arg }
   if arg.isMVar then
-    let mvarId := arg.mvarId!
-    if let some mvarErrorInfo ← getMVarErrorInfo? mvarId then
-      registerMVarErrorInfo { mvarErrorInfo with argName? := argName }
+    registerMVarArgName arg.mvarId! argName
 
 /--
   Elaborate the given `Arg` and add it to the result. See `addNewArg`.
@@ -833,9 +831,7 @@ private def elabArg (arg : Arg) (argExpectedType : Expr) : M Expr := do
 /-- Save information for producing error messages. -/
 def saveArgInfo (arg : Expr) (binderName : Name) : M Unit := do
   if arg.isMVar then
-    let mvarId := arg.mvarId!
-    if let some mvarErrorInfo ← getMVarErrorInfo? mvarId then
-      registerMVarErrorInfo { mvarErrorInfo with argName? := binderName }
+    registerMVarArgName arg.mvarId! binderName
 
 /-- Create an implicit argument using the given `BinderInfo`. -/
 def mkImplicitArg (argExpectedType : Expr) (bi : BinderInfo) : M Expr := do

src/Lean/Elab/Binders.lean

@@ -782,6 +782,9 @@ def elabLetDeclCore (stx : Syntax) (expectedType? : Option Expr) (useLetExpr : B
 @[builtin_term_elab «let_tmp»] def elabLetTmpDecl : TermElab :=
   fun stx expectedType? => elabLetDeclCore stx expectedType? (useLetExpr := true) (elabBodyFirst := false) (usedLetOnly := true)
 
-builtin_initialize registerTraceClass `Elab.let
+builtin_initialize
+  registerTraceClass `Elab.let
+  registerTraceClass `Elab.let.decl
+  registerTraceClass `Elab.autoParam
 
 end Lean.Elab.Term

src/Lean/Elab/BuiltinCommand.lean

@@ -229,7 +229,7 @@ private def replaceBinderAnnotation (binder : TSyntax ``Parser.Term.bracketedBin
 @[builtin_command_elab «variable»] def elabVariable : CommandElab
   | `(variable $binders*) => do
     -- Try to elaborate `binders` for sanity checking
-    runTermElabM fun _ => Term.withAutoBoundImplicit <|
+    runTermElabM fun _ => Term.withSynthesize <| Term.withAutoBoundImplicit <|
       Term.elabBinders binders fun _ => pure ()
     for binder in binders do
       let binders ← replaceBinderAnnotation binder
@@ -262,16 +262,22 @@ def elabCheckCore (ignoreStuckTC : Bool) : CommandElab
 
 @[builtin_command_elab Lean.Parser.Command.check] def elabCheck : CommandElab := elabCheckCore (ignoreStuckTC := true)
 
-@[builtin_command_elab Lean.Parser.Command.reduce] def elabReduce : CommandElab
-  | `(#reduce%$tk $term) => withoutModifyingEnv <| runTermElabM fun _ => Term.withDeclName `_reduce do
-    let e ← Term.elabTerm term none
-    Term.synthesizeSyntheticMVarsNoPostponing
-    let e ← Term.levelMVarToParam (← instantiateMVars e)
-    -- TODO: add options or notation for setting the following parameters
-    withTheReader Core.Context (fun ctx => { ctx with options := ctx.options.setBool `smartUnfolding false }) do
-      let e ← withTransparency (mode := TransparencyMode.all) <| reduce e (skipProofs := false) (skipTypes := false)
-      logInfoAt tk e
+@[builtin_command_elab Lean.reduceCmd] def elabReduce : CommandElab
+  | `(#reduce%$tk $term) => go tk term
+  | `(#reduce%$tk (proofs := true) $term) => go tk term (skipProofs := false)
+  | `(#reduce%$tk (types := true) $term) => go tk term (skipTypes := false)
+  | `(#reduce%$tk (proofs := true) (types := true) $term) => go tk term (skipProofs := false) (skipTypes := false)
   | _ => throwUnsupportedSyntax
+where
+  go (tk : Syntax) (term : Syntax) (skipProofs := true) (skipTypes := true) : CommandElabM Unit :=
+    withoutModifyingEnv <| runTermElabM fun _ => Term.withDeclName `_reduce do
+      let e ← Term.elabTerm term none
+      Term.synthesizeSyntheticMVarsNoPostponing
+      let e ← Term.levelMVarToParam (← instantiateMVars e)
+      -- TODO: add options or notation for setting the following parameters
+      withTheReader Core.Context (fun ctx => { ctx with options := ctx.options.setBool `smartUnfolding false }) do
+        let e ← withTransparency (mode := TransparencyMode.all) <| reduce e (skipProofs := skipProofs) (skipTypes := skipTypes)
+        logInfoAt tk e
 
 def hasNoErrorMessages : CommandElabM Bool := do
   return !(← get).messages.hasErrors
@@ -461,7 +467,9 @@ def elabRunMeta : CommandElab := fun stx =>
   modifyScope fun scope => { scope with opts := options }
 
 @[builtin_macro Lean.Parser.Command.«in»] def expandInCmd : Macro
-  | `($cmd₁ in $cmd₂) => `(section $cmd₁:command $cmd₂ end)
+  | `($cmd₁ in%$tk $cmd₂) =>
+    -- Limit ref variability for incrementality; see Note [Incremental Macros]
+    withRef tk `(section $cmd₁:command $cmd₂ end)
   | _                 => Macro.throwUnsupported
 
 @[builtin_command_elab Parser.Command.addDocString] def elabAddDeclDoc : CommandElab := fun stx => do

src/Lean/Elab/BuiltinTerm.lean

@@ -190,8 +190,18 @@ private def mkFreshTypeMVarFor (expectedType? : Option Expr) : TermElabM Expr :=
     | some val => pure val
     | none     => throwIllFormedSyntax
   let typeMVar ← mkFreshTypeMVarFor expectedType?
-  let u ← getDecLevel typeMVar
-  let mvar ← mkInstMVar (mkApp2 (Lean.mkConst ``OfNat [u]) typeMVar (mkRawNatLit val))
+  let u ← try
+    getDecLevel typeMVar
+  catch ex =>
+    match expectedType? with
+    | some expectedType =>
+      if (← isProp expectedType) then
+        throwError m!"numerals are data in Lean, but the expected type is a proposition{indentExpr expectedType} : Prop"
+      else
+        throwError m!"numerals are data in Lean, but the expected type is universe polymorphic and may be a proposition{indentExpr expectedType} : {← inferType expectedType}"
+    | none => throw ex
+  let extraMsg := m!"numerals are polymorphic in Lean, but the numeral `{val}` cannot be used in a context where the expected type is{indentExpr typeMVar}\ndue to the absence of the instance above"
+  let mvar ← mkInstMVar (mkApp2 (Lean.mkConst ``OfNat [u]) typeMVar (mkRawNatLit val)) extraMsg
   let r := mkApp3 (Lean.mkConst ``OfNat.ofNat [u]) typeMVar (mkRawNatLit val) mvar
   registerMVarErrorImplicitArgInfo mvar.mvarId! stx r
   return r

src/Lean/Elab/CheckTactic.lean

@@ -17,6 +17,7 @@ namespace Lean.Elab.CheckTactic
 
 open Lean.Meta CheckTactic
 open Lean.Elab.Tactic
+open Lean.Elab.Term
 open Lean.Elab.Command
 
 @[builtin_command_elab Lean.Parser.checkTactic]
@@ -24,7 +25,7 @@ def elabCheckTactic : CommandElab := fun stx => do
   let `(#check_tactic $t ~> $result by $tac) := stx | throwUnsupportedSyntax
   withoutModifyingEnv $ do
     runTermElabM $ fun _vars => do
-      let u ← Lean.Elab.Term.elabTerm t none
+      let u ← withSynthesize (postpone := .no) <| Lean.Elab.Term.elabTerm t none
       let type ← inferType u
       let checkGoalType ← mkCheckGoalType u type
       let mvar ← mkFreshExprMVar (.some checkGoalType)

src/Lean/Elab/Command.lean

@@ -338,6 +338,26 @@ instance : MonadRecDepth CommandElabM where
 
 builtin_initialize registerTraceClass `Elab.command
 
+open Language in
+/-- Snapshot after macro expansion of a command. -/
+structure MacroExpandedSnapshot extends Snapshot where
+  /-- The declaration name of the macro. -/
+  macroDecl : Name
+  /-- The expanded syntax tree. -/
+  newStx    : Syntax
+  /-- `State.nextMacroScope` after expansion. -/
+  newNextMacroScope : Nat
+  /-- Whether any traces were present after expansion. -/
+  hasTraces : Bool
+  /--
+  Follow-up elaboration snapshots, one per command if `newStx` is a sequence of commands.
+  -/
+  next : Array (SnapshotTask DynamicSnapshot)
+deriving TypeName
+open Language in
+instance : ToSnapshotTree MacroExpandedSnapshot where
+  toSnapshotTree s := ⟨s.toSnapshot, s.next.map (·.map (sync := true) toSnapshotTree)⟩
+
 partial def elabCommand (stx : Syntax) : CommandElabM Unit := do
   withLogging <| withRef stx <| withIncRecDepth <| withFreshMacroScope do
     match stx with
@@ -345,8 +365,8 @@ partial def elabCommand (stx : Syntax) : CommandElabM Unit := do
       if k == nullKind then
         -- list of commands => elaborate in order
         -- The parser will only ever return a single command at a time, but syntax quotations can return multiple ones
-        -- TODO: support incrementality at least for some cases such as expansions of
-        -- `set_option in` or `def a.b`
+        -- Incrementality is currently limited to the common case where the sequence is the direct
+        -- output of a macro, see below.
         withoutCommandIncrementality true do
           args.forM elabCommand
       else withTraceNode `Elab.command (fun _ => return stx) (tag :=
@@ -358,7 +378,55 @@ partial def elabCommand (stx : Syntax) : CommandElabM Unit := do
           withInfoTreeContext (mkInfoTree := mkInfoTree decl stx) do
             let stxNew ← liftMacroM <| liftExcept stxNew?
             withMacroExpansion stx stxNew do
-              elabCommand stxNew
+              -- Support incrementality; see also Note [Incremental Macros]
+              if let some snap := (←read).snap? then
+                -- Unpack nested commands; see `MacroExpandedSnapshot.next`
+                let cmds := if stxNew.isOfKind nullKind then stxNew.getArgs else #[stxNew]
+                let nextMacroScope := (← get).nextMacroScope
+                let hasTraces := (← getTraceState).traces.size > 0
+                let oldSnap? := do
+                  let oldSnap ← snap.old?
+                  let oldSnap ← oldSnap.val.get.toTyped? MacroExpandedSnapshot
+                  guard <| oldSnap.macroDecl == decl && oldSnap.newNextMacroScope == nextMacroScope
+                  -- check absence of traces; see Note [Incremental Macros]
+                  guard <| !oldSnap.hasTraces && !hasTraces
+                  return oldSnap
+                let oldCmds? := oldSnap?.map fun old =>
+                  if old.newStx.isOfKind nullKind then old.newStx.getArgs else #[old.newStx]
+                Language.withAlwaysResolvedPromises cmds.size fun cmdPromises => do
+                  snap.new.resolve <| .ofTyped {
+                    diagnostics := .empty
+                    macroDecl := decl
+                    newStx := stxNew
+                    newNextMacroScope := nextMacroScope
+                    hasTraces
+                    next := cmdPromises.zipWith cmds fun cmdPromise cmd =>
+                      { range? := cmd.getRange?, task := cmdPromise.result }
+                    : MacroExpandedSnapshot
+                  }
+                  -- After the first command whose syntax tree changed, we must disable
+                  -- incremental reuse
+                  let mut reusedCmds := true
+                  let opts ← getOptions
+                  -- For each command, associate it with new promise and old snapshot, if any, and
+                  -- elaborate recursively
+                  for cmd in cmds, cmdPromise in cmdPromises, i in [0:cmds.size] do
+                    let oldCmd? := oldCmds?.bind (·[i]?)
+                    withReader ({ · with snap? := some {
+                      new := cmdPromise
+                      old? := do
+                        guard reusedCmds
+                        let old ← oldSnap?
+                        return { stx := (← oldCmd?), val := (← old.next[i]?) }
+                    } }) do
+                      elabCommand cmd
+                      -- Resolve promise for commands not supporting incrementality; waiting for
+                      -- `withAlwaysResolvedPromises` to do this could block reporting by later
+                      -- commands
+                      cmdPromise.resolve default
+                    reusedCmds := reusedCmds && oldCmd?.any (·.eqWithInfoAndTraceReuse opts cmd)
+              else
+                elabCommand stxNew
         | _ =>
           match commandElabAttribute.getEntries s.env k with
           | []      =>
@@ -377,6 +445,10 @@ register_builtin_option showPartialSyntaxErrors : Bool := {
   descr    := "show elaboration errors from partial syntax trees (i.e. after parser recovery)"
 }
 
+builtin_initialize
+  registerTraceClass `Elab.info
+  registerTraceClass `Elab.snapshotTree
+
 /--
 `elabCommand` wrapper that should be used for the initial invocation, not for recursive calls after
 macro expansion etc.
@@ -399,6 +471,12 @@ def elabCommandTopLevel (stx : Syntax) : CommandElabM Unit := withRef stx do pro
     let mut msgs := (← get).messages
     for tree in (← getInfoTrees) do
       trace[Elab.info] (← tree.format)
+    if let some snap := (← read).snap? then
+      -- We can assume that the root command snapshot is not involved in parallelism yet, so this
+      -- should be true iff the command supports incrementality
+      if (← IO.hasFinished snap.new.result) then
+        trace[Elab.snapshotTree]
+          Language.ToSnapshotTree.toSnapshotTree snap.new.result.get |>.format
     modify fun st => { st with
       messages := initMsgs ++ msgs
       infoState := { st.infoState with trees := initInfoTrees ++ st.infoState.trees }

src/Lean/Elab/DeclUtil.lean

@@ -9,19 +9,25 @@ import Lean.Meta.Check
 
 namespace Lean.Meta
 
-def forallTelescopeCompatibleAux {α} (k : Array Expr → Expr → Expr → MetaM α) : Nat → Expr → Expr → Array Expr → MetaM α
+def forallTelescopeCompatibleAux (k : Array Expr → Expr → Expr → MetaM α) : Nat → Expr → Expr → Array Expr → MetaM α
   | 0, type₁, type₂, xs   => k xs type₁ type₂
   | i+1, type₁, type₂, xs => do
     let type₁ ← whnf type₁
     let type₂ ← whnf type₂
     match type₁, type₂ with
-    | Expr.forallE n₁ d₁ b₁ c₁, Expr.forallE n₂ d₂ b₂ c₂ =>
-      unless n₁ == n₂ do
-        throwError "parameter name mismatch '{n₁}', expected '{n₂}'"
-      unless (← isDefEq d₁ d₂) do
-        throwError "parameter '{n₁}' {← mkHasTypeButIsExpectedMsg d₁ d₂}"
+    | .forallE n₁ d₁ b₁ c₁, .forallE n₂ d₂ b₂ c₂ =>
+      -- Remark: we use `mkIdent` to ensure macroscopes do not leak into error messages
       unless c₁ == c₂ do
-        throwError "binder annotation mismatch at parameter '{n₁}'"
+        throwError "binder annotation mismatch at parameter '{mkIdent n₁}'"
+      /-
+      Remark: recall that users may suppress parameter names for instance implicit arguments.
+      A fresh name (with macro scopes) is generated in this case. Thus, we allow the names
+      to be different in this case. See issue #4310.
+      -/
+      unless n₁ == n₂ || (c₁.isInstImplicit && n₁.hasMacroScopes && n₂.hasMacroScopes) do
+        throwError "parameter name mismatch '{mkIdent n₁}', expected '{mkIdent n₂}'"
+      unless (← isDefEq d₁ d₂) do
+        throwError "parameter '{mkIdent n₁}' {← mkHasTypeButIsExpectedMsg d₁ d₂}"
       withLocalDecl n₁ c₁ d₁ fun x =>
         let type₁ := b₁.instantiate1 x
         let type₂ := b₂.instantiate1 x
@@ -30,7 +36,7 @@ def forallTelescopeCompatibleAux {α} (k : Array Expr → Expr → Expr → Meta
 
 /-- Given two forall-expressions `type₁` and `type₂`, ensure the first `numParams` parameters are compatible, and
     then execute `k` with the parameters and remaining types. -/
-def forallTelescopeCompatible {α m} [Monad m] [MonadControlT MetaM m] (type₁ type₂ : Expr) (numParams : Nat) (k : Array Expr → Expr → Expr → m α) : m α :=
+def forallTelescopeCompatible [Monad m] [MonadControlT MetaM m] (type₁ type₂ : Expr) (numParams : Nat) (k : Array Expr → Expr → Expr → m α) : m α :=
   controlAt MetaM fun runInBase =>
     forallTelescopeCompatibleAux (fun xs type₁ type₂ => runInBase $ k xs type₁ type₂) numParams type₁ type₂ #[]
 
@@ -69,9 +75,11 @@ def sortDeclLevelParams (scopeParams : List Name) (allUserParams : List Name) (u
   match allUserParams.find? fun u => !usedParams.contains u && !scopeParams.elem u with
   | some u => throw s!"unused universe parameter '{u}'"
   | none   =>
+    -- Recall that `allUserParams` (like `scopeParams`) are in reverse order. That is, the last declared universe is the first element of the list.
+    -- The following `foldl` will reverse the elements and produce a list of universe levels using the user given order.
     let result := allUserParams.foldl (fun result levelName => if usedParams.elem levelName then levelName :: result else result) []
     let remaining := usedParams.filter (fun levelParam => !allUserParams.elem levelParam)
     let remaining := remaining.qsort Name.lt
-    pure $ result ++ remaining.toList
+    return result ++ remaining.toList
 
 end Lean.Elab

src/Lean/Elab/Declaration.lean

@@ -188,34 +188,42 @@ def elabClassInductive (modifiers : Modifiers) (stx : Syntax) : CommandElabM Uni
   let v ← classInductiveSyntaxToView modifiers stx
   elabInductiveViews #[v]
 
+/--
+Macro that expands a declaration with a complex name into an explicit `namespace` block.
+Implementing this step as a macro means that reuse checking is handled by `elabCommand`.
+ -/
+@[builtin_macro Lean.Parser.Command.declaration]
+def expandNamespacedDeclaration : Macro := fun stx => do
+  match (← expandDeclNamespace? stx) with
+  | some (ns, newStx) => do
+    -- Limit ref variability for incrementality; see Note [Incremental Macros]
+    let declTk := stx[1][0]
+    let ns := mkIdentFrom declTk ns
+    withRef declTk `(namespace $ns $(⟨newStx⟩) end $ns)
+  | none => Macro.throwUnsupported
+
 @[builtin_command_elab declaration, builtin_incremental]
 def elabDeclaration : CommandElab := fun stx => do
-  match (← liftMacroM <| expandDeclNamespace? stx) with
-  | some (ns, newStx) => do
-    let ns := mkIdentFrom stx ns
-    let newStx ← `(namespace $ns $(⟨newStx⟩) end $ns)
-    withMacroExpansion stx newStx <| elabCommand newStx
-  | none => do
-    let decl     := stx[1]
-    let declKind := decl.getKind
-    if isDefLike decl then
-      -- only case implementing incrementality currently
-      elabMutualDef #[stx]
-    else withoutCommandIncrementality true do
-      if declKind == ``Lean.Parser.Command.«axiom» then
-        let modifiers ← elabModifiers stx[0]
-        elabAxiom modifiers decl
-      else if declKind == ``Lean.Parser.Command.«inductive» then
-        let modifiers ← elabModifiers stx[0]
-        elabInductive modifiers decl
-      else if declKind == ``Lean.Parser.Command.classInductive then
-        let modifiers ← elabModifiers stx[0]
-        elabClassInductive modifiers decl
-      else if declKind == ``Lean.Parser.Command.«structure» then
-        let modifiers ← elabModifiers stx[0]
-        elabStructure modifiers decl
-      else
-        throwError "unexpected declaration"
+  let decl     := stx[1]
+  let declKind := decl.getKind
+  if isDefLike decl then
+    -- only case implementing incrementality currently
+    elabMutualDef #[stx]
+  else withoutCommandIncrementality true do
+    if declKind == ``Lean.Parser.Command.«axiom» then
+      let modifiers ← elabModifiers stx[0]
+      elabAxiom modifiers decl
+    else if declKind == ``Lean.Parser.Command.«inductive» then
+      let modifiers ← elabModifiers stx[0]
+      elabInductive modifiers decl
+    else if declKind == ``Lean.Parser.Command.classInductive then
+      let modifiers ← elabModifiers stx[0]
+      elabClassInductive modifiers decl
+    else if declKind == ``Lean.Parser.Command.«structure» then
+      let modifiers ← elabModifiers stx[0]
+      elabStructure modifiers decl
+    else
+      throwError "unexpected declaration"
 
 /-- Return true if all elements of the mutual-block are inductive declarations. -/
 private def isMutualInductive (stx : Syntax) : Bool :=
@@ -306,6 +314,10 @@ def expandMutualElement : Macro := fun stx => do
   let mut elemsNew := #[]
   let mut modified := false
   for elem in stx[1].getArgs do
+    -- Don't trigger the `expandNamespacedDecl` macro, the namespace is handled by the mutual def
+    -- elaborator directly instead
+    if elem.isOfKind ``Parser.Command.declaration then
+      continue
     match (← expandMacro? elem) with
     | some elemNew => elemsNew := elemsNew.push elemNew; modified := true
     | none         => elemsNew := elemsNew.push elem

src/Lean/Elab/Deriving/DecEq.lean

@@ -182,8 +182,7 @@ def mkDecEqEnum (declName : Name) : CommandElabM Unit := do
       fun x y =>
         if h : x.toCtorIdx = y.toCtorIdx then
           -- We use `rfl` in the following proof because the first script fails for unit-like datatypes due to etaStruct.
-          -- Temporarily avoiding tactic `have` for bootstrapping
-          isTrue (by first | refine_lift have aux := congrArg $ofNatIdent h; ?_; rw [$auxThmIdent:ident, $auxThmIdent:ident] at aux; assumption | rfl)
+          isTrue (by first | have aux := congrArg $ofNatIdent h; rw [$auxThmIdent:ident, $auxThmIdent:ident] at aux; assumption | rfl)
         else
           isFalse fun h => by subst h; contradiction
   )

src/Lean/Elab/Frontend.lean

@@ -132,9 +132,6 @@ def process (input : String) (env : Environment) (opts : Options) (fileName : Op
   let s ← IO.processCommands inputCtx { : Parser.ModuleParserState } (Command.mkState env {} opts)
   pure (s.commandState.env, s.commandState.messages)
 
-builtin_initialize
-  registerTraceClass `Elab.info
-
 @[export lean_run_frontend]
 def runFrontend
     (input : String)

src/Lean/Elab/Level.lean

@@ -78,7 +78,7 @@ partial def elabLevel (stx : Syntax) : LevelElabM Level := withRef stx do
       if (← read).autoBoundImplicit && isValidAutoBoundLevelName paramName (relaxedAutoImplicit.get (← read).options) then
         modify fun s => { s with levelNames := paramName :: s.levelNames }
       else
-        throwError "unknown universe level '{paramName}'"
+        throwError "unknown universe level '{mkIdent paramName}'"
     return mkLevelParam paramName
   else if kind == `Lean.Parser.Level.addLit then
     let lvl ← elabLevel (stx.getArg 0)

src/Lean/Elab/MutualDef.lean

@@ -141,17 +141,12 @@ private def elabHeaders (views : Array DefView)
     for view in views, ⟨shortDeclName, declName, levelNames⟩ in expandedDeclIds,
         tacPromise in tacPromises, bodyPromise in bodyPromises do
       let mut reusableResult? := none
+      let mut oldBodySnap? := none
+      let mut oldTacSnap? := none
       if let some snap := view.headerSnap? then
         -- by the `DefView.headerSnap?` invariant, safe to reuse results at this point, so let's
         -- wait for them!
         if let some old := snap.old?.bind (·.val.get) then
-          let (tacStx?, newTacTask?) ← mkTacTask view.value tacPromise
-          snap.new.resolve <| some { old with
-            tacStx?
-            tacSnap? := newTacTask?
-            bodyStx := view.value
-            bodySnap := mkBodyTask view.value bodyPromise
-          }
           -- Transition from `DefView.snap?` to `DefViewElabHeader.tacSnap?` invariant: if all
           -- headers and all previous bodies could be reused, then the state at the *start* of the
           -- top-level tactic block (if any) is unchanged
@@ -160,27 +155,19 @@ private def elabHeaders (views : Array DefView)
           -- headers and all previous bodies could be reused and this body syntax is unchanged, then
           -- we can reuse the result
           reuseBody := reuseBody &&
-            view.value.structRangeEqWithTraceReuse (← getOptions) old.bodyStx
-          let header := { old.view, view with
-            -- We should only forward the promise if we are actually waiting on the corresponding
-            -- task; otherwise, diagnostics assigned to it will be lost
-            tacSnap? := guard newTacTask?.isSome *> some {
-              old? := do
-                guard reuseTac
-                some ⟨(← old.tacStx?), (← old.tacSnap?)⟩
-              new := tacPromise
-            }
-            bodySnap? := some {
-              -- no syntax guard to store, we already did the necessary checks
-              old? := guard reuseBody *> pure ⟨.missing, old.bodySnap⟩
-              new := bodyPromise
-            }
-          }
-          reusableResult? := some (header, old.state)
+            view.value.eqWithInfoAndTraceReuse (← getOptions) old.bodyStx
+          -- no syntax guard to store, we already did the necessary checks
+          oldBodySnap? := guard reuseBody *> pure ⟨.missing, old.bodySnap⟩
+          oldTacSnap? := do
+              guard reuseTac
+              some ⟨(← old.tacStx?), (← old.tacSnap?)⟩
+          let newHeader : DefViewElabHeader := { view, old.view with
+            bodySnap? := none, tacSnap? := none }  -- filled below
+          reusableResult? := some (newHeader, old.state)
         else
           reuseBody := false
 
-      let header ← withRestoreOrSaveFull reusableResult? fun save => do
+      let mut (newHeader, newState) ← withRestoreOrSaveFull reusableResult? do
         withRef view.headerRef do
         addDeclarationRanges declName view.ref  -- NOTE: this should be the full `ref`
         applyAttributesAt declName view.modifiers.attrs .beforeElaboration
@@ -219,29 +206,29 @@ private def elabHeaders (views : Array DefView)
               declName, shortDeclName, type, levelNames, binderIds
               numParams := xs.size
             }
-            let mut newHeader : DefViewElabHeader := { view, newHeader with
+            let newHeader : DefViewElabHeader := { view, newHeader with
               bodySnap? := none, tacSnap? := none }
-            if let some snap := view.headerSnap? then
-              let (tacStx?, newTacTask?) ← mkTacTask view.value tacPromise
-              snap.new.resolve <| some {
-                diagnostics :=
-                  (← Language.Snapshot.Diagnostics.ofMessageLog (← Core.getAndEmptyMessageLog))
-                view := newHeader.toDefViewElabHeaderData
-                state := (← save)
-                tacStx?
-                tacSnap? := newTacTask?
-                bodyStx := view.value
-                bodySnap := mkBodyTask view.value bodyPromise
-              }
-              newHeader := { newHeader with
-                -- We should only forward the promise if we are actually waiting on the
-                -- corresponding task; otherwise, diagnostics assigned to it will be lost
-                tacSnap? := guard newTacTask?.isSome *> some { old? := none, new := tacPromise }
-                bodySnap? := some { old? := none, new := bodyPromise }
-              }
             check headers newHeader
             return newHeader
-      headers := headers.push header
+      if let some snap := view.headerSnap? then
+        let (tacStx?, newTacTask?) ← mkTacTask view.value tacPromise
+        snap.new.resolve <| some {
+          diagnostics :=
+            (← Language.Snapshot.Diagnostics.ofMessageLog (← Core.getAndEmptyMessageLog))
+          view := newHeader.toDefViewElabHeaderData
+          state := newState
+          tacStx?
+          tacSnap? := newTacTask?
+          bodyStx := view.value
+          bodySnap := mkBodyTask view.value bodyPromise
+        }
+        newHeader := { newHeader with
+          -- We should only forward the promise if we are actually waiting on the
+          -- corresponding task; otherwise, diagnostics assigned to it will be lost
+          tacSnap? := guard newTacTask?.isSome *> some { old? := oldTacSnap?, new := tacPromise }
+          bodySnap? := some { old? := oldBodySnap?, new := bodyPromise }
+        }
+      headers := headers.push newHeader
     return headers
 where
   getBodyTerm? (stx : Syntax) : Option Syntax :=
@@ -357,7 +344,7 @@ private def elabFunValues (headers : Array DefViewElabHeader) : TermElabM (Array
               tacSnap.new.resolve oldTacSnap.val.get
           reusableResult? := some (old.value, old.state)
 
-    withRestoreOrSaveFull reusableResult? fun save => do
+    let (val, state) ← withRestoreOrSaveFull reusableResult? do
       withDeclName header.declName <| withLevelNames header.levelNames do
       let valStx ← liftMacroM <| declValToTerm header.value
       forallBoundedTelescope header.type header.numParams fun xs type => do
@@ -371,15 +358,15 @@ private def elabFunValues (headers : Array DefViewElabHeader) : TermElabM (Array
         -- NOTE: without this `instantiatedMVars`, `mkLambdaFVars` may leave around a redex that
         -- leads to more section variables being included than necessary
         let val ← instantiateMVars val
-        let val ← mkLambdaFVars xs val
-        if let some snap := header.bodySnap? then
-          snap.new.resolve <| some {
-            diagnostics :=
-              (← Language.Snapshot.Diagnostics.ofMessageLog (← Core.getAndEmptyMessageLog))
-            state := (← save)
-            value := val
-          }
-        return val
+        mkLambdaFVars xs val
+    if let some snap := header.bodySnap? then
+      snap.new.resolve <| some {
+        diagnostics :=
+          (← Language.Snapshot.Diagnostics.ofMessageLog (← Core.getAndEmptyMessageLog))
+        state
+        value := val
+      }
+    return val
 
 private def collectUsed (headers : Array DefViewElabHeader) (values : Array Expr) (toLift : List LetRecToLift)
     : StateRefT CollectFVars.State MetaM Unit := do
@@ -990,7 +977,7 @@ def elabMutualDef (ds : Array Syntax) : CommandElabM Unit := do
             -- blocking wait, `HeadersParsedSnapshot` (and hopefully others) should be quick
             let old ← old.val.get.toTyped? DefsParsedSnapshot
             let oldParsed ← old.defs[i]?
-            guard <| fullHeaderRef.structRangeEqWithTraceReuse opts oldParsed.fullHeaderRef
+            guard <| fullHeaderRef.eqWithInfoAndTraceReuse opts oldParsed.fullHeaderRef
             -- no syntax guard to store, we already did the necessary checks
             return ⟨.missing, oldParsed.headerProcessedSnap⟩
           new := headerPromise
@@ -1006,5 +993,8 @@ def elabMutualDef (ds : Array Syntax) : CommandElabM Unit := do
       snap.new.resolve <| .ofTyped { defs, diagnostics := .empty : DefsParsedSnapshot }
     runTermElabM fun vars => Term.elabMutualDef vars views
 
+builtin_initialize
+  registerTraceClass `Elab.definition.mkClosure
+
 end Command
 end Lean.Elab

src/Lean/Elab/PatternVar.lean

@@ -47,8 +47,51 @@ structure State where
 
 abbrev M := StateRefT State TermElabM
 
-private def throwCtorExpected {α} : M α :=
-  throwError "invalid pattern, constructor or constant marked with '[match_pattern]' expected"
+private def throwCtorExpected {α} (ident : Option Syntax) : M α := do
+  let message : MessageData :=
+    "invalid pattern, constructor or constant marked with '[match_pattern]' expected"
+  let some idStx := ident | throwError message
+  let name := idStx.getId
+  if let .anonymous := name then throwError message
+  let env ← getEnv
+  let mut candidates : Array Name := #[]
+  for (c, _) in env.constants do
+    if isPrivateName c then continue
+    if !(name.isSuffixOf c) then continue
+    if env.isConstructor c || hasMatchPatternAttribute env c then
+      candidates := candidates.push c
+
+  if candidates.size = 0 then
+    throwError message
+  else if h : candidates.size = 1 then
+    throwError message ++ m!"\n\nSuggestion: '{candidates[0]}' is similar"
+  else
+    let sorted := candidates.qsort (·.toString < ·.toString)
+    let diff :=
+      if candidates.size > 10 then [m!" (or {candidates.size - 10} others)"]
+      else []
+    let suggestions : MessageData := .group <|
+      .joinSep ((sorted.extract 0 10 |>.toList |>.map (showName env)) ++ diff)
+        ("," ++ Format.line)
+    throwError message ++ .group ("\n\nSuggestions:" ++ .nestD (Format.line ++ suggestions))
+where
+  -- Create some `MessageData` for a name that shows it without an `@`, but with the metadata that
+  -- makes infoview hovers and the like work. This technique only works because the names are known
+  -- to be global constants, so we don't need the local context.
+  showName (env : Environment) (n : Name) : MessageData :=
+      let params :=
+        env.constants.find?' n |>.map (·.levelParams.map Level.param) |>.getD []
+      .ofFormatWithInfos {
+        fmt := "'" ++ .tag 0 (format n) ++ "'",
+        infos :=
+          .fromList [(0, .ofTermInfo {
+            lctx := .empty,
+            expr := .const n params,
+            stx := .ident .none (toString n).toSubstring n [.decl n []],
+            elaborator := `Delab,
+            expectedType? := none
+          })] _
+      }
 
 private def throwInvalidPattern {α} : M α :=
   throwError "invalid pattern"
@@ -169,9 +212,9 @@ partial def collect (stx : Syntax) : M Syntax := withRef stx <| withFreshMacroSc
     -- Check whether the `binop%` operator is marked with `[match_pattern]`,
     -- We must check that otherwise Lean will accept operators that are not tagged with this annotation.
     let some (.const fName _) ← resolveId? stx[1] "pattern"
-      | throwCtorExpected
+      | throwCtorExpected none
     unless hasMatchPatternAttribute (← getEnv) fName do
-      throwCtorExpected
+      throwCtorExpected none
     let lhs ← collect stx[2]
     let rhs ← collect stx[3]
     return stx.setArg 2 lhs |>.setArg 3 rhs
@@ -255,7 +298,7 @@ where
             processCtor stx
           else
             processVar stx
-        | none => throwCtorExpected
+        | none => throwCtorExpected (some stx)
       | _ => processVar stx
 
   pushNewArg (accessible : Bool) (ctx : Context) (arg : Arg) : M Context := do
@@ -307,7 +350,7 @@ where
       | `($fId:ident)  => pure (fId, false)
       | `(@$fId:ident) => pure (fId, true)
       | _              => throwError "identifier expected"
-    let some (Expr.const fName _) ← resolveId? fId "pattern" (withInfo := true) | throwCtorExpected
+    let some (Expr.const fName _) ← resolveId? fId "pattern" (withInfo := true) | throwCtorExpected (some fId)
     let fInfo ← getConstInfo fName
     let paramDecls ← forallTelescopeReducing fInfo.type fun xs _ => xs.mapM fun x => do
       let d ← getFVarLocalDecl x
@@ -321,7 +364,7 @@ where
         processCtorAppContext
           { funId := fId, explicit := explicit, ctorVal? := none, paramDecls := paramDecls, namedArgs := namedArgs, args := args, ellipsis := ellipsis }
       else
-        throwCtorExpected
+        throwCtorExpected (some fId)
 
 def main (alt : MatchAltView) : M MatchAltView := do
   let patterns ← alt.patterns.mapM fun p => do

src/Lean/Elab/PreDefinition/WF/TerminationArgument.lean

@@ -86,21 +86,23 @@ def TerminationArgument.delab (termArg : TerminationArgument) : MetaM (TSyntax `
     let e ← mkLambdaFVars ys[termArg.arity - termArg.extraParams:] e -- undo overshooting by lambdaTelescope
     pure (← delabCore e (delab := go termArg.extraParams #[])).1
   where
-    go : Nat → TSyntaxArray [`ident, `Lean.Parser.Term.hole] → DelabM (TSyntax ``terminationBy)
+    go : Nat → TSyntaxArray `ident → DelabM (TSyntax ``terminationBy)
     | 0, vars => do
+      let stxBody ← Delaborator.delab
+      let hole : TSyntax `Lean.Parser.Term.hole ← `(hole|_)
+
+      -- any variable not mentioned syntatically (it may appear in the `Expr`, so do not just use
+      -- `e.bindingBody!.hasLooseBVar`) should be delaborated as a hole.
+      let vars  : TSyntaxArray [`ident, `Lean.Parser.Term.hole] :=
+        Array.map (fun (i : Ident) => if hasIdent i.getId stxBody then i else hole) vars
       -- drop trailing underscores
       let mut vars := vars
       while ! vars.isEmpty && vars.back.raw.isOfKind ``hole do vars := vars.pop
       if vars.isEmpty then
-        `(terminationBy|termination_by $(← Delaborator.delab))
+        `(terminationBy|termination_by $stxBody)
       else
-        `(terminationBy|termination_by $vars* => $(← Delaborator.delab))
+        `(terminationBy|termination_by $vars* => $stxBody)
     | i+1, vars => do
       let e ← getExpr
       unless e.isLambda do return ← go 0 vars -- should not happen
-
-      -- Delaborate unused parameters with `_`
-      if e.bindingBody!.hasLooseBVar 0 then
-        withBindingBodyUnusedName fun n => go i (vars.push ⟨n⟩)
-      else
-        descend e.bindingBody! 1 (go i (vars.push (← `(hole|_))))
+      withBindingBodyUnusedName fun n => go i (vars.push ⟨n⟩)

src/Lean/Elab/Print.lean

@@ -153,7 +153,7 @@ private def printAxiomsOf (constName : Name) : CommandElabM Unit := do
   if s.axioms.isEmpty then
     logInfo m!"'{constName}' does not depend on any axioms"
   else
-    logInfo m!"'{constName}' depends on axioms: {s.axioms.toList}"
+    logInfo m!"'{constName}' depends on axioms: {s.axioms.qsort Name.lt |>.toList}"
 
 @[builtin_command_elab «printAxioms»] def elabPrintAxioms : CommandElab
   | `(#print%$tk axioms $id) => withRef tk do

src/Lean/Elab/Quotation.lean

@@ -690,5 +690,6 @@ builtin_initialize
   registerTraceClass `Elab.match_syntax
   registerTraceClass `Elab.match_syntax.alt (inherited := true)
   registerTraceClass `Elab.match_syntax.result (inherited := true)
+  registerTraceClass `Elab.match_syntax.onMatch
 
 end Lean.Elab.Term.Quotation

src/Lean/Elab/StructInst.lean

@@ -957,6 +957,8 @@ private def elabStructInstAux (stx : Syntax) (expectedType? : Option Expr) (sour
     else
       elabStructInstAux stx expectedType? sourceView
 
-builtin_initialize registerTraceClass `Elab.struct
+builtin_initialize
+  registerTraceClass `Elab.struct
+  registerTraceClass `Elab.struct.modifyOp
 
 end Lean.Elab.Term.StructInst

src/Lean/Elab/Syntax.lean

@@ -442,4 +442,7 @@ def strLitToPattern (stx: Syntax) : MacroM Syntax :=
   | some str => return mkAtomFrom stx str
   | none     => Macro.throwUnsupported
 
+builtin_initialize
+  registerTraceClass `Elab.defaultInstance
+
 end Lean.Elab.Command

src/Lean/Elab/SyntheticMVars.lean

@@ -59,10 +59,10 @@ private def resumePostponed (savedContext : SavedContext) (stx : Syntax) (mvarId
 /--
   Similar to `synthesizeInstMVarCore`, but makes sure that `instMVar` local context and instances
   are used. It also logs any error message produced. -/
-private def synthesizePendingInstMVar (instMVar : MVarId) : TermElabM Bool :=
+private def synthesizePendingInstMVar (instMVar : MVarId) (extraErrorMsg? : Option MessageData := none): TermElabM Bool :=
   instMVar.withContext do
     try
-      synthesizeInstMVarCore instMVar
+      synthesizeInstMVarCore instMVar (extraErrorMsg? := extraErrorMsg?)
     catch
       | ex@(.error ..) => logException ex; return true
       | _              => unreachable!
@@ -180,7 +180,7 @@ private def synthesizeSomeUsingDefaultPrio (prio : Nat) : TermElabM Bool := do
     | mvarId :: pendingMVars =>
       let some mvarDecl ← getSyntheticMVarDecl? mvarId | visit pendingMVars (mvarId :: pendingMVarsNew)
       match mvarDecl.kind with
-      | .typeClass =>
+      | .typeClass .. => -- TODO: use `errorMsg?` in `typeClass`.
         if (← withRef mvarDecl.stx <| synthesizeUsingDefaultPrio mvarId prio) then
           modify fun s => { s with pendingMVars := pendingMVars.reverse ++ pendingMVarsNew }
           return true
@@ -211,12 +211,13 @@ def reportStuckSyntheticMVar (mvarId : MVarId) (ignoreStuckTC := false) : TermEl
   let some mvarSyntheticDecl ← getSyntheticMVarDecl? mvarId | return ()
   withRef mvarSyntheticDecl.stx do
     match mvarSyntheticDecl.kind with
-    | .typeClass =>
+    | .typeClass extraErrorMsg? =>
+      let extraErrorMsg := extraMsgToMsg extraErrorMsg?
       unless ignoreStuckTC do
          mvarId.withContext do
           let mvarDecl ← getMVarDecl mvarId
           unless (← MonadLog.hasErrors) do
-            throwError "typeclass instance problem is stuck, it is often due to metavariables{indentExpr mvarDecl.type}"
+            throwError "typeclass instance problem is stuck, it is often due to metavariables{indentExpr mvarDecl.type}{extraErrorMsg}"
     | .coe header expectedType e f? =>
       mvarId.withContext do
         throwTypeMismatchError header expectedType (← inferType e) e f?
@@ -346,7 +347,10 @@ mutual
               -- view even though it is synthetic while a node like `tacticCode` never is (#1990)
               withTacticInfoContext tacticCode[0] do
                 withNarrowedArgTacticReuse (argIdx := 1) (evalTactic ·) tacticCode
-            synthesizeSyntheticMVars (postpone := .no)
+            -- Pending tactic mvars may escape from `evalTactic` to here (#4436), so make sure
+            -- incrementality is disabled so they cannot be confused for top-level tactic blocks
+            withoutTacticIncrementality true do
+              synthesizeSyntheticMVars (postpone := .no)
           unless remainingGoals.isEmpty do
             if report then
               reportUnsolvedGoals remainingGoals
@@ -365,7 +369,7 @@ mutual
     let some mvarSyntheticDecl ← getSyntheticMVarDecl? mvarId | return true -- The metavariable has already been synthesized
     withRef mvarSyntheticDecl.stx do
     match mvarSyntheticDecl.kind with
-    | .typeClass => synthesizePendingInstMVar mvarId
+    | .typeClass extraErrorMsg? => synthesizePendingInstMVar mvarId extraErrorMsg?
     | .coe _header? expectedType e _f? => mvarId.withContext do
       if (← withDefault do isDefEq (← inferType e) expectedType) then
         -- Types may be defeq now due to mvar assignments, type class

src/Lean/Elab/Tactic/Basic.lean

@@ -97,14 +97,14 @@ def SavedState.restore (b : SavedState) (restoreInfo := false) : TacticM Unit :=
   set b.tactic
 
 @[specialize, inherit_doc Core.withRestoreOrSaveFull]
-def withRestoreOrSaveFull (reusableResult? : Option (α × SavedState))
-    (cont : TacticM SavedState → TacticM α) : TacticM α := do
+def withRestoreOrSaveFull (reusableResult? : Option (α × SavedState)) (act : TacticM α) :
+    TacticM (α × SavedState) := do
   if let some (_, state) := reusableResult? then
     set state.tactic
   let reusableResult? := reusableResult?.map (fun (val, state) => (val, state.term))
-  controlAt TermElabM fun runInBase =>
-    Term.withRestoreOrSaveFull reusableResult? fun restore =>
-      runInBase <| cont (return { term := (← restore), tactic := (← get) })
+  let (a, term) ← controlAt TermElabM fun runInBase => do
+    Term.withRestoreOrSaveFull reusableResult? <| runInBase act
+  return (a, { term, tactic := (← get) })
 
 protected def getCurrMacroScope : TacticM MacroScope := do pure (← readThe Core.Context).currMacroScope
 protected def getMainModule     : TacticM Name       := do pure (← getEnv).mainModule
@@ -201,6 +201,37 @@ where
           withReader ({ · with elaborator := m.declName }) do
             withTacticInfoContext stx do
               let stx' ← adaptMacro m.value stx
+              -- Support incrementality; see also Note [Incremental Macros]
+              if evalFns.isEmpty && ms.isEmpty then  -- Only try incrementality in one branch
+                if let some snap := (← readThe Term.Context).tacSnap? then
+                  let nextMacroScope := (← getThe Core.State).nextMacroScope
+                  let traceState ← getTraceState
+                  let old? := do
+                    let old ← snap.old?
+                    -- If the kind is equal, we can assume the old version was a macro as well
+                    guard <| old.stx.isOfKind stx.getKind
+                    let state ← old.val.get.data.finished.get.state?
+                    guard <| state.term.meta.core.nextMacroScope == nextMacroScope
+                    -- check absence of traces; see Note [Incremental Macros]
+                    guard <| state.term.meta.core.traceState.traces.size == 0
+                    guard <| traceState.traces.size == 0
+                    return old.val.get
+                  Language.withAlwaysResolvedPromise fun promise => do
+                    -- Store new unfolding in the snapshot tree
+                    snap.new.resolve <| .mk {
+                      stx := stx'
+                      diagnostics := .empty
+                      finished := .pure { state? := (← Tactic.saveState) }
+                    } #[{ range? := stx'.getRange?, task := promise.result }]
+                    -- Update `tacSnap?` to old unfolding
+                    withTheReader Term.Context ({ · with tacSnap? := some {
+                      new := promise
+                      old? := do
+                        let old ← old?
+                        return ⟨old.data.stx, (← old.next.get? 0)⟩
+                    } }) do
+                      evalTactic stx'
+                  return
               evalTactic stx'
         catch ex => handleEx s failures ex (expandEval s ms evalFns)
 

src/Lean/Elab/Tactic/BuiltinTactic.lean

@@ -21,10 +21,10 @@ namespace Lean.Elab.Tactic
 open Meta
 open Parser.Tactic
 
-@[builtin_tactic withAnnotateState] def evalWithAnnotateState : Tactic
-  | `(tactic| with_annotate_state $stx $t) =>
-    withTacticInfoContext stx (evalTactic t)
-  | _ => throwUnsupportedSyntax
+@[builtin_tactic withAnnotateState, builtin_incremental] def evalWithAnnotateState : Tactic :=
+  fun stx =>
+    withTacticInfoContext stx[1] do
+    Term.withNarrowedArgTacticReuse (argIdx := 2) evalTactic stx
 
 @[builtin_tactic Lean.Parser.Tactic.«done»] def evalDone : Tactic := fun _ =>
   done
@@ -71,7 +71,7 @@ where
         -- `tac` must be unchanged given the narrow above; let's reuse `finished`'s state!
         let oldParsed := old.val.get
         if let some state := oldParsed.data.finished.get.state? then
-          reusableResult? := some (state, state)
+          reusableResult? := some ((), state)
           -- only allow `next` reuse in this case
           oldNext? := oldParsed.next.get? 1 |>.map (⟨old.stx, ·⟩)
 
@@ -90,12 +90,11 @@ where
               {
                 range? := stxs |>.getRange?
                 task := next.result }]
-            let state ← withRestoreOrSaveFull reusableResult? fun save => do
+            let (_, state) ← withRestoreOrSaveFull reusableResult? do
               -- set up nested reuse; `evalTactic` will check for `isIncrementalElab`
               withTheReader Term.Context ({ · with
                   tacSnap? := some { old? := oldInner?, new := inner } }) do
                 evalTactic tac
-              save
             finished.resolve { state? := state }
 
         withTheReader Term.Context ({ · with tacSnap? := some {
@@ -114,8 +113,8 @@ where
 @[builtin_tactic seq1] def evalSeq1 : Tactic := fun stx =>
   evalSepTactics stx[0]
 
-@[builtin_tactic paren] def evalParen : Tactic := fun stx =>
-  evalTactic stx[1]
+@[builtin_tactic paren, builtin_incremental] def evalParen : Tactic :=
+  Term.withNarrowedArgTacticReuse 1 evalTactic
 
 def isCheckpointableTactic (arg : Syntax) : TacticM Bool := do
   -- TODO: make it parametric
@@ -205,12 +204,12 @@ def evalTacticCDot : Tactic := fun stx => do
     withInfoContext (pure ()) initInfo
     Term.withNarrowedArgTacticReuse (argIdx := 1) evalTactic stx
 
-@[builtin_tactic Parser.Tactic.focus] def evalFocus : Tactic := fun stx => do
+@[builtin_tactic Parser.Tactic.focus, builtin_incremental] def evalFocus : Tactic := fun stx => do
   let mkInfo ← mkInitialTacticInfo stx[0]
   focus do
     -- show focused state on `focus`
     withInfoContext (pure ()) mkInfo
-    evalTactic stx[1]
+    Term.withNarrowedArgTacticReuse (argIdx := 1) evalTactic stx
 
 private def getOptRotation (stx : Syntax) : Nat :=
   if stx.isNone then 1 else stx[0].toNat

src/Lean/Elab/Tactic/Calc.lean

@@ -15,11 +15,11 @@ open Meta
 def evalCalc : Tactic := fun stx => withMainContext do
   let steps : TSyntax ``calcSteps := ⟨stx[1]⟩
   let (val, mvarIds) ← withCollectingNewGoalsFrom (tagSuffix := `calc) do
-    let target ← getMainTarget
+    let target := (← getMainTarget).consumeMData
     let tag ← getMainTag
     runTermElab do
     let mut val ← Term.elabCalcSteps steps
-    let mut valType ← inferType val
+    let mut valType ← instantiateMVars (← inferType val)
     unless (← isDefEq valType target) do
       let rec throwFailed :=
         throwError "'calc' tactic failed, has type{indentExpr valType}\nbut it is expected to have type{indentExpr target}"

src/Lean/Elab/Tactic/Omega/Frontend.lean

@@ -217,8 +217,10 @@ partial def asLinearComboImpl (e : Expr) : OmegaM (LinearCombo × OmegaM Expr ×
         match groundInt? b with
         | some _ => rewrite e (mkApp2 (.const ``Int.pow_succ []) b k')
         | none => mkAtomLinearCombo e
-  | (``Nat.cast, #[.const ``Int [], i, n]) =>
-      handleNatCast e i n
+  | (``Nat.cast, #[α, i, n]) =>
+      match_expr α with
+      | Int => handleNatCast e i n
+      | _ => mkAtomLinearCombo e
   | (``Prod.fst, #[α, β, p]) => match p with
     | .app (.app (.app (.app (.const ``Prod.mk [u, v]) _) _) x) y =>
       rewrite e (mkApp4 (.const ``Prod.fst_mk [u, v]) α x β y)
@@ -423,65 +425,69 @@ partial def addFact (p : MetaProblem) (h : Expr) : OmegaM (MetaProblem × Nat) :
       else
         return (p, 0)
     | .app _ _ =>
-      match t.getAppFnArgs with
-      | (``Eq, #[.const ``Int [], x, y]) =>
-        match y.int? with
-        | some 0 => pure (← p.addIntEquality h x, 1)
-        | _ => p.addFact (mkApp3 (.const ``Int.sub_eq_zero_of_eq []) x y h)
-      | (``LE.le, #[.const ``Int [], _, x, y]) =>
-        match x.int? with
-        | some 0 => pure (← p.addIntInequality h y, 1)
-        | _ => p.addFact (mkApp3 (.const ``Int.sub_nonneg_of_le []) y x h)
-      | (``LT.lt, #[.const ``Int [], _, x, y]) =>
-        p.addFact (mkApp3 (.const ``Int.add_one_le_of_lt []) x y h)
-      | (``GT.gt, #[.const ``Int [], _, x, y]) =>
-        p.addFact (mkApp3 (.const ``Int.lt_of_gt []) x y h)
-      | (``GE.ge, #[.const ``Int [], _, x, y]) =>
-        p.addFact (mkApp3 (.const ``Int.le_of_ge []) x y h)
-      | (``GT.gt, #[.const ``Nat [], _, x, y]) =>
-        p.addFact (mkApp3 (.const ``Nat.lt_of_gt []) x y h)
-      | (``GE.ge, #[.const ``Nat [], _, x, y]) =>
-        p.addFact (mkApp3 (.const ``Nat.le_of_ge []) x y h)
-      | (``Ne, #[.const ``Nat [], x, y]) =>
-        p.addFact (mkApp3 (.const ``Nat.lt_or_gt_of_ne []) x y h)
-      | (``Not, #[P]) => match ← pushNot h P with
+      match_expr t with
+      | Eq α x y =>
+        match_expr α with
+        | Int =>
+          match y.int? with
+          | some 0 => pure (← p.addIntEquality h x, 1)
+          | _ => p.addFact (mkApp3 (.const ``Int.sub_eq_zero_of_eq []) x y h)
+        | Nat => p.addFact (mkApp3 (.const ``Int.ofNat_congr []) x y h)
+        | Fin n => p.addFact (mkApp4 (.const ``Fin.val_congr []) n x y h)
+        | _ => pure (p, 0)
+      | LE.le α _ x y =>
+        match_expr α with
+        | Int =>
+          match x.int? with
+          | some 0 => pure (← p.addIntInequality h y, 1)
+          | _ => p.addFact (mkApp3 (.const ``Int.sub_nonneg_of_le []) y x h)
+        | Nat => p.addFact (mkApp3 (.const ``Int.ofNat_le_of_le []) x y h)
+        | Fin n => p.addFact (mkApp4 (.const ``Fin.val_le_of_le []) n x y h)
+        | _ => pure (p, 0)
+      | LT.lt α _ x y =>
+        match_expr α with
+        | Int => p.addFact (mkApp3 (.const ``Int.add_one_le_of_lt []) x y h)
+        | Nat => p.addFact (mkApp3 (.const ``Int.ofNat_lt_of_lt []) x y h)
+        | Fin n => p.addFact (mkApp4 (.const ``Fin.val_add_one_le_of_lt []) n x y h)
+        | _ => pure (p, 0)
+      | GT.gt α _ x y =>
+        match_expr α with
+        | Int => p.addFact (mkApp3 (.const ``Int.lt_of_gt []) x y h)
+        | Nat => p.addFact (mkApp3 (.const ``Nat.lt_of_gt []) x y h)
+        | Fin n => p.addFact (mkApp4 (.const ``Fin.val_add_one_le_of_gt []) n x y h)
+        | _ => pure (p, 0)
+      | GE.ge α _ x y =>
+        match_expr α with
+        | Int => p.addFact (mkApp3 (.const ``Int.le_of_ge []) x y h)
+        | Nat => p.addFact (mkApp3 (.const ``Nat.le_of_ge []) x y h)
+        | Fin n => p.addFact (mkApp4 (.const ``Fin.val_le_of_ge []) n x y h)
+        | _ => pure (p, 0)
+      | Ne α x y =>
+        match_expr α with
+        | Int => p.addFact (mkApp3 (.const ``Int.lt_or_gt_of_ne []) x y h)
+        | Nat => p.addFact (mkApp3 (.const ``Nat.lt_or_gt_of_ne []) x y h)
+        | _ => pure (p, 0)
+      | Dvd.dvd α _ k x =>
+        match_expr α with
+        | Int => p.addFact (mkApp3 (.const ``Int.emod_eq_zero_of_dvd []) k x h)
+        | Nat => p.addFact (mkApp3 (.const ``Nat.mod_eq_zero_of_dvd []) k x h)
+        | _ => pure (p, 0)
+      | Not P => match ← pushNot h P with
         | none => return (p, 0)
         | some h' => p.addFact h'
-      | (``Eq, #[.const ``Nat [], x, y]) =>
-        p.addFact (mkApp3 (.const ``Int.ofNat_congr []) x y h)
-      | (``LT.lt, #[.const ``Nat [], _, x, y]) =>
-        p.addFact (mkApp3 (.const ``Int.ofNat_lt_of_lt []) x y h)
-      | (``LE.le, #[.const ``Nat [], _, x, y]) =>
-        p.addFact (mkApp3 (.const ``Int.ofNat_le_of_le []) x y h)
-      | (``Ne, #[.const ``Int [], x, y]) =>
-        p.addFact (mkApp3 (.const ``Int.lt_or_gt_of_ne []) x y h)
-      | (``Prod.Lex, _) => p.addFact (← mkAppM ``Prod.of_lex #[h])
-      | (``Dvd.dvd, #[.const ``Nat [], _, k, x]) =>
-        p.addFact (mkApp3 (.const ``Nat.mod_eq_zero_of_dvd []) k x h)
-      | (``Dvd.dvd, #[.const ``Int [], _, k, x]) =>
-        p.addFact (mkApp3 (.const ``Int.emod_eq_zero_of_dvd []) k x h)
-      | (``Eq, #[.app (.const ``Fin []) n, x, y]) =>
-        p.addFact (mkApp4 (.const ``Fin.val_congr []) n x y h)
-      | (``LE.le, #[.app (.const ``Fin []) n, _, x, y]) =>
-        p.addFact (mkApp4 (.const ``Fin.val_le_of_le []) n x y h)
-      | (``LT.lt, #[.app (.const ``Fin []) n, _, x, y]) =>
-        p.addFact (mkApp4 (.const ``Fin.val_add_one_le_of_lt []) n x y h)
-      | (``GE.ge, #[.app (.const ``Fin []) n, _, x, y]) =>
-        p.addFact (mkApp4 (.const ``Fin.val_le_of_ge []) n x y h)
-      | (``GT.gt, #[.app (.const ``Fin []) n, _, x, y]) =>
-        p.addFact (mkApp4 (.const ``Fin.val_add_one_le_of_gt []) n x y h)
-      | (``And, #[t₁, t₂]) => do
+      | Prod.Lex _ _ _ _ _ _ => p.addFact (← mkAppM ``Prod.of_lex #[h])
+      | And t₁ t₂ => do
           let (p₁, n₁) ← p.addFact (mkApp3 (.const ``And.left []) t₁ t₂ h)
           let (p₂, n₂) ← p₁.addFact (mkApp3 (.const ``And.right []) t₁ t₂ h)
           return (p₂, n₁ + n₂)
-      | (``Exists, #[α, P]) =>
+      | Exists α P =>
         p.addFact (mkApp3 (.const ``Exists.choose_spec [← getLevel α]) α P h)
-      | (``Subtype, #[α, P]) =>
+      | Subtype α P =>
         p.addFact (mkApp3 (.const ``Subtype.property [← getLevel α]) α P h)
-      | (``Iff, #[P₁, P₂]) =>
+      | Iff P₁ P₂ =>
         p.addFact (mkApp4 (.const ``Decidable.and_or_not_and_not_of_iff [])
           P₁ P₂ (.app (.const ``Classical.propDecidable []) P₂) h)
-      | (``Or, #[_, _]) =>
+      | Or _ _ =>
         if (← cfg).splitDisjunctions then
           return ({ p with disjunctions := p.disjunctions.insert h }, 1)
         else
@@ -537,10 +543,11 @@ def formatErrorMessage (p : Problem) : OmegaM MessageData := do
       division, and modular remainder by constants."
     else
       let as ← atoms
-      let mask ← mentioned p.constraints
-      let names ← varNames mask
-      return m!"a possible counterexample may satisfy the constraints\n" ++
-        m!"{prettyConstraints names p.constraints}\nwhere\n{prettyAtoms names as mask}"
+      return .ofLazyM (es := as) do
+        let mask ← mentioned as p.constraints
+        let names ← varNames mask
+        return m!"a possible counterexample may satisfy the constraints\n" ++
+          m!"{prettyConstraints names p.constraints}\nwhere\n{prettyAtoms names as mask}"
   else
     -- formatErrorMessage should not be used in this case
     return "it is trivially solvable"
@@ -593,8 +600,8 @@ where
         (if Int.natAbs c = 1 then names[i]! else s!"{c.natAbs}*{names[i]!}"))
       |> String.join
 
-  mentioned (constraints : HashMap Coeffs Fact) : OmegaM (Array Bool) := do
-    let initMask := Array.mkArray (← getThe State).atoms.size false
+  mentioned (atoms : Array Expr) (constraints : HashMap Coeffs Fact) : MetaM (Array Bool) := do
+    let initMask := Array.mkArray atoms.size false
     return constraints.fold (init := initMask) fun mask coeffs _ =>
       coeffs.enum.foldl (init := mask) fun mask (i, c) =>
         if c = 0 then mask else mask.set! i true

src/Lean/Elab/Tactic/Simp.lean

@@ -153,7 +153,8 @@ inductive ResolveSimpIdResult where
   Elaborate extra simp theorems provided to `simp`. `stx` is of the form `"[" simpTheorem,* "]"`
   If `eraseLocal == true`, then we consider local declarations when resolving names for erased theorems (`- id`),
   this option only makes sense for `simp_all` or `*` is used.
-  Try to recover from errors as much as possible so that users keep seeing the current goal.
+  When `recover := true`, try to recover from errors as much as possible so that users keep seeing
+  the current goal.
 -/
 def elabSimpArgs (stx : Syntax) (ctx : Simp.Context) (simprocs : Simp.SimprocsArray) (eraseLocal : Bool) (kind : SimpKind) : TacticM ElabSimpArgsResult := do
   if stx.isNone then
@@ -223,7 +224,11 @@ def elabSimpArgs (stx : Syntax) (ctx : Simp.Context) (simprocs : Simp.SimprocsAr
             starArg := true
           else
             throwUnsupportedSyntax
-        catch ex => logException ex
+        catch ex =>
+          if (← read).recover then
+            logException ex
+          else
+            throw ex
       return { ctx := { ctx with simpTheorems := thmsArray.set! 0 thms }, simprocs, starArg }
 where
   isSimproc? (e : Expr) : MetaM (Option Name) := do

src/Lean/Elab/Tactic/Split.lean

@@ -21,12 +21,12 @@ open Meta
       throwErrorAt stx[2] "'split' tactic failed, select a single target to split"
     if simplifyTarget then
       liftMetaTactic fun mvarId => do
-       let some mvarIds ← splitTarget? mvarId | Meta.throwTacticEx `split mvarId
+       let some mvarIds ← splitTarget? mvarId | Meta.throwTacticEx `split mvarId "consider using `set_option trace.split.failure true`"
         return mvarIds
     else
       let fvarId ← getFVarId hyps[0]!
       liftMetaTactic fun mvarId => do
-        let some mvarIds ← splitLocalDecl? mvarId fvarId | Meta.throwTacticEx `split mvarId
+        let some mvarIds ← splitLocalDecl? mvarId fvarId | Meta.throwTacticEx `split mvarId "consider using `set_option trace.split.failure true`"
         return mvarIds
   | Location.wildcard =>
     liftMetaTactic fun mvarId => do
@@ -34,7 +34,7 @@ open Meta
       for fvarId in fvarIds do
         if let some mvarIds ← splitLocalDecl? mvarId fvarId then
           return mvarIds
-      let some mvarIds ← splitTarget? mvarId | Meta.throwTacticEx `split mvarId
+      let some mvarIds ← splitTarget? mvarId | Meta.throwTacticEx `split mvarId "consider using `set_option trace.split.failure true`"
       return mvarIds
 
 end Lean.Elab.Tactic

src/Lean/Elab/Term.lean

@@ -30,8 +30,12 @@ structure SavedContext where
 
 /-- We use synthetic metavariables as placeholders for pending elaboration steps. -/
 inductive SyntheticMVarKind where
-  /-- Use typeclass resolution to synthesize value for metavariable. -/
-  | typeClass
+  /--
+  Use typeclass resolution to synthesize value for metavariable.
+  If `extraErrorMsg?` is `some msg`, `msg` contains additional information to include in error messages
+  regarding type class synthesis failure.
+  -/
+  | typeClass (extraErrorMsg? : Option MessageData)
   /-- Use coercion to synthesize value for the metavariable.
   if `f?` is `some f`, we produce an application type mismatch error message.
   Otherwise, if `header?` is `some header`, we generate the error `(header ++ "has type" ++ eType ++ "but it is expected to have type" ++ expectedType)`
@@ -43,9 +47,15 @@ inductive SyntheticMVarKind where
   | postponed (ctx : SavedContext)
   deriving Inhabited
 
+/--
+Convert an "extra" optional error message into a message `"\n{msg}"` (if `some msg`) and `MessageData.nil` (if `none`)
+-/
+def extraMsgToMsg (extraErrorMsg? : Option MessageData) : MessageData :=
+  if let some msg := extraErrorMsg? then m!"\n{msg}" else .nil
+
 instance : ToString SyntheticMVarKind where
   toString
-    | .typeClass    => "typeclass"
+    | .typeClass .. => "typeclass"
     | .coe ..       => "coe"
     | .tactic ..    => "tactic"
     | .postponed .. => "postponed"
@@ -81,7 +91,6 @@ structure MVarErrorInfo where
   mvarId    : MVarId
   ref       : Syntax
   kind      : MVarErrorKind
-  argName?  : Option Name := none
   deriving Inhabited
 
 /--
@@ -109,7 +118,20 @@ structure State where
   levelNames        : List Name       := []
   syntheticMVars    : MVarIdMap SyntheticMVarDecl := {}
   pendingMVars      : List MVarId := {}
-  mvarErrorInfos    : MVarIdMap MVarErrorInfo := {}
+  /-- List of errors associated to a metavariable that are shown to the user if the metavariable could not be fully instantiated -/
+  mvarErrorInfos    : List MVarErrorInfo := []
+  /--
+    `mvarArgNames` stores the argument names associated to metavariables.
+    These are used in combination with `mvarErrorInfos` for throwing errors about metavariables that could not be fully instantiated.
+    For example when elaborating `List _`, the argument name of the placeholder will be `α`.
+
+    While elaborating an application, `mvarArgNames` is set for each metavariable argument, using the available argument name.
+    This may happen before or after the `mvarErrorInfos` is set for the same metavariable.
+
+    We used to store the argument names in `mvarErrorInfos`, updating the `MVarErrorInfos` to add the argument name when it is available,
+    but this doesn't work if the argument name is available _before_ the `mvarErrorInfos` is set for that metavariable.
+  -/
+  mvarArgNames      : MVarIdMap Name := {}
   letRecsToLift     : List LetRecToLift := []
   deriving Inhabited
 
@@ -306,14 +328,14 @@ def SavedState.restore (s : SavedState) (restoreInfo : Bool := false) : TermElab
     setInfoState infoState
 
 @[specialize, inherit_doc Core.withRestoreOrSaveFull]
-def withRestoreOrSaveFull (reusableResult? : Option (α × SavedState))
-    (cont : TermElabM SavedState → TermElabM α) : TermElabM α := do
+def withRestoreOrSaveFull (reusableResult? : Option (α × SavedState)) (act : TermElabM α) :
+    TermElabM (α × SavedState) := do
   if let some (_, state) := reusableResult? then
     set state.elab
   let reusableResult? := reusableResult?.map (fun (val, state) => (val, state.meta))
-  controlAt MetaM fun runInBase =>
-    Meta.withRestoreOrSaveFull reusableResult? fun restore =>
-      runInBase <| cont (return { meta := (← restore), «elab» := (← get) })
+  let (a, meta) ← controlAt MetaM fun runInBase => do
+    Meta.withRestoreOrSaveFull reusableResult? <| runInBase act
+  return (a, { meta, «elab» := (← get) })
 
 instance : MonadBacktrack SavedState TermElabM where
   saveState      := Term.saveState
@@ -323,7 +345,7 @@ instance : MonadBacktrack SavedState TermElabM where
 Manages reuse information for nested tactics by `split`ting given syntax into an outer and inner
 part. `act` is then run on the inner part but with reuse information adjusted as following:
 * If the old (from `tacSnap?`'s `SyntaxGuarded.stx`) and new (from `stx`) outer syntax are not
-  identical according to `Syntax.structRangeEq`, reuse is disabled.
+  identical according to `Syntax.eqWithInfo`, reuse is disabled.
 * Otherwise, the old syntax as stored in `tacSnap?` is updated to the old *inner* syntax.
 * In any case, we also use `withRef` on the inner syntax to avoid leakage of the outer syntax into
   `act` via this route.
@@ -339,7 +361,7 @@ def withNarrowedTacticReuse [Monad m] [MonadExceptOf Exception m] [MonadWithRead
   withTheReader Term.Context (fun ctx => { ctx with tacSnap? := ctx.tacSnap?.map fun tacSnap =>
     { tacSnap with old? := tacSnap.old?.bind fun old => do
       let (oldOuter, oldInner) := split old.stx
-      guard <| outer.structRangeEqWithTraceReuse opts oldOuter
+      guard <| outer.eqWithInfoAndTraceReuse opts oldOuter
       return { old with stx := oldInner }
     }
   }) do
@@ -616,7 +638,7 @@ def registerSyntheticMVarWithCurrRef (mvarId : MVarId) (kind : SyntheticMVarKind
   registerSyntheticMVar (← getRef) mvarId kind
 
 def registerMVarErrorInfo (mvarErrorInfo : MVarErrorInfo) : TermElabM Unit :=
-  modify fun s => { s with mvarErrorInfos := s.mvarErrorInfos.insert mvarErrorInfo.mvarId mvarErrorInfo }
+  modify fun s => { s with mvarErrorInfos := mvarErrorInfo :: s.mvarErrorInfos }
 
 def registerMVarErrorHoleInfo (mvarId : MVarId) (ref : Syntax) : TermElabM Unit :=
   registerMVarErrorInfo { mvarId, ref, kind := .hole }
@@ -627,14 +649,14 @@ def registerMVarErrorImplicitArgInfo (mvarId : MVarId) (ref : Syntax) (app : Exp
 def registerMVarErrorCustomInfo (mvarId : MVarId) (ref : Syntax) (msgData : MessageData) : TermElabM Unit := do
   registerMVarErrorInfo { mvarId, ref, kind := .custom msgData }
 
-def getMVarErrorInfo? (mvarId : MVarId) : TermElabM (Option MVarErrorInfo) := do
-  return (← get).mvarErrorInfos.find? mvarId
-
 def registerCustomErrorIfMVar (e : Expr) (ref : Syntax) (msgData : MessageData) : TermElabM Unit :=
   match e.getAppFn with
   | Expr.mvar mvarId => registerMVarErrorCustomInfo mvarId ref msgData
   | _ => pure ()
 
+def registerMVarArgName (mvarId : MVarId) (argName : Name) : TermElabM Unit :=
+  modify fun s => { s with mvarArgNames := s.mvarArgNames.insert mvarId argName }
+
 /--
   Auxiliary method for reporting errors of the form "... contains metavariables ...".
   This kind of error is thrown, for example, at `Match.lean` where elaboration
@@ -650,22 +672,22 @@ def MVarErrorInfo.logError (mvarErrorInfo : MVarErrorInfo) (extraMsg? : Option M
   match mvarErrorInfo.kind with
   | MVarErrorKind.implicitArg app => do
     let app ← instantiateMVars app
-    let msg := addArgName "don't know how to synthesize implicit argument"
+    let msg ← addArgName "don't know how to synthesize implicit argument"
     let msg := msg ++ m!"{indentExpr app.setAppPPExplicitForExposingMVars}" ++ Format.line ++ "context:" ++ Format.line ++ MessageData.ofGoal mvarErrorInfo.mvarId
     logErrorAt mvarErrorInfo.ref (appendExtra msg)
   | MVarErrorKind.hole => do
-    let msg := addArgName "don't know how to synthesize placeholder" " for argument"
+    let msg ← addArgName "don't know how to synthesize placeholder" " for argument"
     let msg := msg ++ Format.line ++ "context:" ++ Format.line ++ MessageData.ofGoal mvarErrorInfo.mvarId
     logErrorAt mvarErrorInfo.ref (MessageData.tagged `Elab.synthPlaceholder <| appendExtra msg)
   | MVarErrorKind.custom msg =>
     logErrorAt mvarErrorInfo.ref (appendExtra msg)
 where
-  /-- Append `mvarErrorInfo` argument name (if available) to the message.
+  /-- Append the argument name (if available) to the message.
       Remark: if the argument name contains macro scopes we do not append it. -/
-  addArgName (msg : MessageData) (extra : String := "") : MessageData :=
-    match mvarErrorInfo.argName? with
-    | none => msg
-    | some argName => if argName.hasMacroScopes then msg else msg ++ extra ++ m!" '{argName}'"
+  addArgName (msg : MessageData) (extra : String := "") : TermElabM MessageData := do
+    match (← get).mvarArgNames.find? mvarErrorInfo.mvarId with
+    | none => return msg
+    | some argName => return if argName.hasMacroScopes then msg else msg ++ extra ++ m!" '{argName}'"
 
   appendExtra (msg : MessageData) : MessageData :=
     match extraMsg? with
@@ -687,7 +709,7 @@ def logUnassignedUsingErrorInfos (pendingMVarIds : Array MVarId) (extraMsg? : Op
     let mut hasNewErrors := false
     let mut alreadyVisited : MVarIdSet := {}
     let mut errors : Array MVarErrorInfo := #[]
-    for (_, mvarErrorInfo) in (← get).mvarErrorInfos do
+    for mvarErrorInfo in (← get).mvarErrorInfos do
       let mvarId := mvarErrorInfo.mvarId
       unless alreadyVisited.contains mvarId do
         alreadyVisited := alreadyVisited.insert mvarId
@@ -727,7 +749,8 @@ def mkExplicitBinder (ident : Syntax) (type : Syntax) : Syntax :=
 def levelMVarToParam (e : Expr) (except : LMVarId → Bool := fun _ => false) : TermElabM Expr := do
   let levelNames ← getLevelNames
   let r := (← getMCtx).levelMVarToParam (fun n => levelNames.elem n) except e `u 1
-  setLevelNames (levelNames ++ r.newParamNames.toList)
+  -- Recall that the most recent universe is the first element of the field `levelNames`.
+  setLevelNames (r.newParamNames.reverse.toList ++ levelNames)
   setMCtx r.mctx
   return r.expr
 
@@ -747,30 +770,35 @@ def mkFreshIdent [Monad m] [MonadQuotation m] (ref : Syntax) (canonical := false
 private def applyAttributesCore
     (declName : Name) (attrs : Array Attribute)
     (applicationTime? : Option AttributeApplicationTime) : TermElabM Unit := do profileitM Exception "attribute application" (← getOptions) do
-  for attr in attrs do
-    withRef attr.stx do withLogging do
-    let env ← getEnv
-    match getAttributeImpl env attr.name with
-    | Except.error errMsg => throwError errMsg
-    | Except.ok attrImpl  =>
-      let runAttr := attrImpl.add declName attr.stx attr.kind
-      let runAttr := do
-        -- not truly an elaborator, but a sensible target for go-to-definition
-        let elaborator := attrImpl.ref
-        if (← getInfoState).enabled && (← getEnv).contains elaborator then
-          withInfoContext (mkInfo := return .ofCommandInfo { elaborator, stx := attr.stx }) do
-            try runAttr
-            finally if attr.stx[0].isIdent || attr.stx[0].isAtom then
-              -- Add an additional node over the leading identifier if there is one to make it look more function-like.
-              -- Do this last because we want user-created infos to take precedence
-              pushInfoLeaf <| .ofCommandInfo { elaborator, stx := attr.stx[0] }
-        else
-          runAttr
-      match applicationTime? with
-      | none => runAttr
-      | some applicationTime =>
-        if applicationTime == attrImpl.applicationTime then
-          runAttr
+  /-
+  Remark: if the declaration has syntax errors, `declName` may be `.anonymous` see issue #4309
+  In this case, we skip attribute application.
+  -/
+  unless declName == .anonymous do
+    for attr in attrs do
+      withRef attr.stx do withLogging do
+      let env ← getEnv
+      match getAttributeImpl env attr.name with
+      | Except.error errMsg => throwError errMsg
+      | Except.ok attrImpl  =>
+        let runAttr := attrImpl.add declName attr.stx attr.kind
+        let runAttr := do
+          -- not truly an elaborator, but a sensible target for go-to-definition
+          let elaborator := attrImpl.ref
+          if (← getInfoState).enabled && (← getEnv).contains elaborator then
+            withInfoContext (mkInfo := return .ofCommandInfo { elaborator, stx := attr.stx }) do
+              try runAttr
+              finally if attr.stx[0].isIdent || attr.stx[0].isAtom then
+                -- Add an additional node over the leading identifier if there is one to make it look more function-like.
+                -- Do this last because we want user-created infos to take precedence
+                pushInfoLeaf <| .ofCommandInfo { elaborator, stx := attr.stx[0] }
+          else
+            runAttr
+        match applicationTime? with
+        | none => runAttr
+        | some applicationTime =>
+          if applicationTime == attrImpl.applicationTime then
+            runAttr
 
 /-- Apply given attributes **at** a given application time -/
 def applyAttributesAt (declName : Name) (attrs : Array Attribute) (applicationTime : AttributeApplicationTime) : TermElabM Unit :=
@@ -852,8 +880,12 @@ def containsPendingMVar (e : Expr) : MetaM Bool := do
   Return `true` if the instance was synthesized successfully, and `false` if
   the instance contains unassigned metavariables that are blocking the type class
   resolution procedure. Throw an exception if resolution or assignment irrevocably fails.
+
+  If `extraErrorMsg?` is not none, it contains additional information that should be attached
+  to type class synthesis failures.
 -/
-def synthesizeInstMVarCore (instMVar : MVarId) (maxResultSize? : Option Nat := none) : TermElabM Bool := do
+def synthesizeInstMVarCore (instMVar : MVarId) (maxResultSize? : Option Nat := none) (extraErrorMsg? : Option MessageData := none): TermElabM Bool := do
+  let extraErrorMsg := extraMsgToMsg extraErrorMsg?
   let instMVarDecl ← getMVarDecl instMVar
   let type := instMVarDecl.type
   let type ← instantiateMVars type
@@ -886,18 +918,18 @@ def synthesizeInstMVarCore (instMVar : MVarId) (maxResultSize? : Option Nat := n
         let oldValType ← inferType oldVal
         let valType ← inferType val
         unless (← isDefEq oldValType valType) do
-          throwError "synthesized type class instance type is not definitionally equal to expected type, synthesized{indentExpr val}\nhas type{indentExpr valType}\nexpected{indentExpr oldValType}"
-        throwError "synthesized type class instance is not definitionally equal to expression inferred by typing rules, synthesized{indentExpr val}\ninferred{indentExpr oldVal}"
+          throwError "synthesized type class instance type is not definitionally equal to expected type, synthesized{indentExpr val}\nhas type{indentExpr valType}\nexpected{indentExpr oldValType}{extraErrorMsg}"
+        throwError "synthesized type class instance is not definitionally equal to expression inferred by typing rules, synthesized{indentExpr val}\ninferred{indentExpr oldVal}{extraErrorMsg}"
     else
       unless (← isDefEq (mkMVar instMVar) val) do
-        throwError "failed to assign synthesized type class instance{indentExpr val}"
+        throwError "failed to assign synthesized type class instance{indentExpr val}{extraErrorMsg}"
     return true
   | .undef => return false -- we will try later
   | .none  =>
     if (← read).ignoreTCFailures then
       return false
     else
-      throwError "failed to synthesize{indentExpr type}\n{useDiagnosticMsg}"
+      throwError "failed to synthesize{indentExpr type}{extraErrorMsg}\n{useDiagnosticMsg}"
 
 def mkCoe (expectedType : Expr) (e : Expr) (f? : Option Expr := none) (errorMsgHeader? : Option String := none) : TermElabM Expr := do
   withTraceNode `Elab.coe (fun _ => return m!"adding coercion for {e} : {← inferType e} =?= {expectedType}") do
@@ -1598,11 +1630,11 @@ def adaptExpander (exp : Syntax → TermElabM Syntax) : TermElab := fun stx expe
   If type class resolution cannot be executed (e.g., it is stuck because of metavariables in `type`),
   register metavariable as a pending one.
 -/
-def mkInstMVar (type : Expr) : TermElabM Expr := do
+def mkInstMVar (type : Expr) (extraErrorMsg? : Option MessageData := none) : TermElabM Expr := do
   let mvar ← mkFreshExprMVar type MetavarKind.synthetic
   let mvarId := mvar.mvarId!
-  unless (← synthesizeInstMVarCore mvarId) do
-    registerSyntheticMVarWithCurrRef mvarId SyntheticMVarKind.typeClass
+  unless (← synthesizeInstMVarCore mvarId (extraErrorMsg? := extraErrorMsg?)) do
+    registerSyntheticMVarWithCurrRef mvarId (.typeClass extraErrorMsg?)
   return mvar
 
 /--
@@ -1804,9 +1836,9 @@ def resolveName' (ident : Syntax) (explicitLevels : List Level) (expectedType? :
       return (c, ids.head!, ids.tail!)
   | _ => throwError "identifier expected"
 
-def resolveId? (stx : Syntax) (kind := "term") (withInfo := false) : TermElabM (Option Expr) :=
+def resolveId? (stx : Syntax) (kind := "term") (withInfo := false) : TermElabM (Option Expr) := withRef stx do
   match stx with
-  | .ident _ _ val preresolved => do
+  | .ident _ _ val preresolved =>
     let rs ← try resolveName stx val preresolved [] catch _ => pure []
     let rs := rs.filter fun ⟨_, projs⟩ => projs.isEmpty
     let fs := rs.map fun (f, _) => f
@@ -1911,4 +1943,7 @@ def isIncrementalElab [Monad m] [MonadEnv m] [MonadLiftT IO m] (decl : Name) : m
 
 export Term (TermElabM)
 
+builtin_initialize
+  registerTraceClass `Elab.implicitForall
+
 end Lean.Elab

src/Lean/Language/Basic.lean

@@ -162,7 +162,7 @@ Always resolving promises involved in the snapshot tree is important to avoid de
 language server.
 -/
 def withAlwaysResolvedPromise [Monad m] [MonadLiftT BaseIO m] [MonadFinally m] [Inhabited α]
-    (act : IO.Promise α → m Unit) : m Unit := do
+    (act : IO.Promise α → m β) : m β := do
   let p ← IO.Promise.new
   try
     act p
@@ -202,6 +202,17 @@ abbrev SnapshotTree.element : SnapshotTree → Snapshot
 abbrev SnapshotTree.children : SnapshotTree → Array (SnapshotTask SnapshotTree)
   | mk _ children => children
 
+/-- Produces debug tree format of given snapshot tree, synchronously waiting on all children. -/
+partial def SnapshotTree.format : SnapshotTree → Format := go none
+where go range? s :=
+  let range := match range? with
+    | some range => f!"{range.start}..{range.stop} "
+    | none => ""
+  let element := f!"{s.element.diagnostics.msgLog.unreported.size} diagnostics"
+  let children := Std.Format.prefixJoin .line <|
+    s.children.toList.map fun c => go c.range? c.get
+  .nestD f!"• {range}{element}{children}"
+
 /--
   Helper class for projecting a heterogeneous hierarchy of snapshot classes to a homogeneous
   representation. -/
@@ -228,7 +239,6 @@ structure DynamicSnapshot where
   val  : Dynamic
   /-- Snapshot tree retrieved from `val` before erasure. -/
   tree : SnapshotTree
-deriving Nonempty
 
 instance : ToSnapshotTree DynamicSnapshot where
   toSnapshotTree s := s.tree
@@ -243,6 +253,9 @@ def DynamicSnapshot.toTyped? (α : Type) [TypeName α] (snap : DynamicSnapshot)
     Option α :=
   snap.val.get? α
 
+instance : Inhabited DynamicSnapshot where
+  default := .ofTyped { diagnostics := .empty : SnapshotLeaf }
+
 /--
   Runs a tree of snapshots to conclusion, incrementally performing `f` on each snapshot in tree
   preorder. -/

src/Lean/Language/Lean.lean

@@ -140,6 +140,26 @@ times, here is a summary of what it has to do to implement incrementality:
     a wildcard pattern
 -/
 
+/-
+# Note [Incremental Macros]
+
+If we have a macro, we can cheaply support incrementality: as a macro is a pure function, if all
+outputs apart from the expanded syntax tree itself are identical in two document versions, we can
+simply delegate reuse detection to the subsequently called elaborator. All we have to do is to
+forward the old unfolding, if any, to it in the appropriate context field and store the new
+unfolding for that purpose in a new snapshot node whose child will be filled by the called
+elaborator. This is currently implemented for command and tactic macros.
+
+Caveat 1: Traces are an additional output of macro expansion but because they are hard to compare
+and should not be active in standard use cases, we disable incrementality if either version produced
+traces.
+
+Caveat 2: As the default `ref` of a macro spans its entire syntax tree and is applied to any token
+created from a quotation, the ref usually has to be changed to a less variable source using
+`withRef` to achieve effective incrementality. See `Elab.Command.expandNamespacedDeclaration` for a
+simple example and the implementation of tactic `have` for a complex example.
+-/
+
 set_option linter.missingDocs true
 
 namespace Lean.Language.Lean

src/Lean/Level.lean

@@ -258,8 +258,7 @@ def ctorToNat : Level → Nat
   | max ..   => 4
   | imax ..  => 5
 
-/- TODO: use well founded recursion. -/
-partial def normLtAux : Level → Nat → Level → Nat → Bool
+def normLtAux : Level → Nat → Level → Nat → Bool
   | succ l₁, k₁, l₂, k₂ => normLtAux l₁ (k₁+1) l₂ k₂
   | l₁, k₁, succ l₂, k₂ => normLtAux l₁ k₁ l₂ (k₂+1)
   | l₁@(max l₁₁ l₁₂), k₁, l₂@(max l₂₁ l₂₂), k₂ =>

src/Lean/Linter.lean

@@ -6,6 +6,7 @@ Authors: Lars König
 prelude
 import Lean.Linter.Util
 import Lean.Linter.Builtin
+import Lean.Linter.ConstructorAsVariable
 import Lean.Linter.Deprecated
 import Lean.Linter.UnusedVariables
 import Lean.Linter.MissingDocs

src/Lean/Linter/ConstructorAsVariable.lean

@@ -0,0 +1,83 @@
+/-
+Copyright (c) 2024 Lean FRO, LLC. All rights reserved.
+Released under Apache 2.0 license as described in the file LICENSE.
+Authors: David Thrane Christiansen
+-/
+prelude
+import Lean.Elab.Command
+import Lean.Linter.Util
+
+set_option linter.missingDocs true
+
+namespace Lean.Linter
+
+open Lean Elab Command
+open Lean.Linter (logLint)
+
+/--
+A linter that warns when bound variable names are the same as constructor names for their types,
+modulo namespaces.
+ -/
+register_builtin_option linter.constructorNameAsVariable : Bool := {
+  defValue := true,
+  descr := "enable the linter that warns when bound variable names are nullary constructor names"
+}
+
+/--
+Reports when bound variables' names overlap with constructor names for their type. This is to warn
+especially new users that they have built a pattern that matches anything, rather than one that
+matches a particular constructor. Use `linter.constructorNameAsVariable` to disable.
+-/
+def constructorNameAsVariable : Linter where
+  run cmdStx := do
+    let some cmdStxRange := cmdStx.getRange?
+      | return
+
+    let infoTrees := (← get).infoState.trees.toArray
+    let warnings : IO.Ref (Lean.HashMap String.Range (Syntax × Name × Name)) ← IO.mkRef {}
+
+    for tree in infoTrees do
+      tree.visitM' (preNode := fun ci info _ => do
+        match info with
+        | .ofTermInfo ti =>
+          match ti.expr with
+          | .fvar id .. =>
+            let some range := info.range? | return
+            if (← warnings.get).contains range then return
+            let .original .. := info.stx.getHeadInfo | return
+            if ti.isBinder then
+              -- This is a local variable declaration.
+              let some ldecl := ti.lctx.find? id | return
+              -- Skip declarations which are outside the command syntax range, like `variable`s
+              -- (it would be confusing to lint these), or those which are macro-generated
+              if !cmdStxRange.contains range.start || ldecl.userName.hasMacroScopes then return
+              let opts := ci.options
+              -- we have to check for the option again here because it can be set locally
+              if !linter.constructorNameAsVariable.get opts then return
+              if let n@(.str .anonymous s) := info.stx.getId then
+                -- Check whether the type is an inductive type, and get its constructors
+                let ty ←
+                  if let some t := ti.expectedType? then pure t
+                  else ti.runMetaM ci (Meta.inferType ti.expr)
+                let ty ← ti.runMetaM ci (instantiateMVars ty >>= Meta.whnf)
+                if let .const tn _ := ty.getAppFn' then
+                  if let some (.inductInfo i) := (← getEnv).find? tn then
+                    for c in i.ctors do
+                      -- Only warn when the constructor has 0 fields. Pattern variables can't be
+                      -- confused with constructors that want arguments.
+                      if let some (.ctorInfo ctorInfo) := (← getEnv).find? c then
+                        if ctorInfo.numFields > 0 then continue
+                      if let .str _ cn := c then
+                        if cn == s then
+                          warnings.modify (·.insert range (info.stx, n, c))
+            else pure ()
+          | _ => pure ()
+        | _ => pure ())
+
+    -- Sort the outputs by position
+    for (_range, declStx, userName, ctorName) in (← warnings.get).toArray.qsort (·.1.start < ·.1.start) do
+      logLint linter.constructorNameAsVariable declStx <|
+        m!"Local variable '{userName}' resembles constructor '{ctorName}' - " ++
+        m!"write '.{userName}' (with a dot) or '{ctorName}' to use the constructor."
+
+builtin_initialize addLinter constructorNameAsVariable

src/Lean/Message.lean

@@ -107,7 +107,8 @@ def lazy (f : PPContext → IO MessageData)
     (hasSyntheticSorry : MetavarContext → Bool := fun _ => false) : MessageData :=
   .ofLazy (hasSyntheticSorry := hasSyntheticSorry) fun ctx? => do
     let msg ← match ctx? with
-      | .none => pure (.ofFormat "(invalid MessageData.lazy, missing context)")
+      | .none =>
+        pure (.ofFormat "(invalid MessageData.lazy, missing context)") -- see `addMessageContext`
       | .some ctx => f ctx
     return Dynamic.mk msg
 
@@ -141,14 +142,31 @@ def mkPPContext (nCtx : NamingContext) (ctx : MessageDataContext) : PPContext :=
 def ofSyntax (stx : Syntax) : MessageData :=
   -- discard leading/trailing whitespace
   let stx := stx.copyHeadTailInfoFrom .missing
-  .lazy fun ctx => ofFormat <$> ppTerm ctx ⟨stx⟩ -- HACK: might not be a term
+  .ofLazy
+    (fun ctx? => do
+      let msg ← ofFormat <$> match ctx? with
+        | .none => pure stx.formatStx
+        | .some ctx => ppTerm ctx ⟨stx⟩ -- HACK: might not be a term
+      return Dynamic.mk msg)
+    (fun _ => false)
 
 def ofExpr (e : Expr) : MessageData :=
-  .lazy (fun ctx => ofFormatWithInfos <$> ppExprWithInfos ctx e)
-        (fun mctx => instantiateMVarsCore mctx e |>.1.hasSyntheticSorry)
+  .ofLazy
+    (fun ctx? => do
+      let msg ← ofFormatWithInfos <$> match ctx? with
+        | .none => pure (format (toString e))
+        | .some ctx => ppExprWithInfos ctx e
+      return Dynamic.mk msg)
+    (fun mctx => instantiateMVarsCore mctx e |>.1.hasSyntheticSorry)
 
 def ofLevel (l : Level) : MessageData :=
-  .lazy fun ctx => ofFormat <$> ppLevel ctx l
+  .ofLazy
+    (fun ctx? => do
+      let msg ← ofFormat <$> match ctx? with
+        | .none => pure (format l)
+        | .some ctx => ppLevel ctx l
+      return Dynamic.mk msg)
+    (fun _ => false)
 
 def ofName (n : Name) : MessageData := ofFormat (format n)
 
@@ -385,6 +403,13 @@ def indentExpr (e : Expr) : MessageData :=
   indentD e
 
 class AddMessageContext (m : Type → Type) where
+  /-- 
+  Without context, a `MessageData` object may be be missing information
+  (e.g. hover info) for pretty printing, or may print an error. Hence,
+  `addMessageContext` should be called on all constructed `MessageData`
+  (e.g. via `m!`) before taking it out of context (e.g. leaving `MetaM` or
+  `CoreM`).
+  -/
   addMessageContext : MessageData → m MessageData
 
 export AddMessageContext (addMessageContext)

src/Lean/Meta/Basic.lean

@@ -418,14 +418,14 @@ def SavedState.restore (b : SavedState) : MetaM Unit := do
   modify fun s => { s with mctx := b.meta.mctx, zetaDeltaFVarIds := b.meta.zetaDeltaFVarIds, postponed := b.meta.postponed }
 
 @[specialize, inherit_doc Core.withRestoreOrSaveFull]
-def withRestoreOrSaveFull (reusableResult? : Option (α × SavedState))
-    (cont : MetaM SavedState → MetaM α) : MetaM α := do
+def withRestoreOrSaveFull (reusableResult? : Option (α × SavedState)) (act : MetaM α) :
+    MetaM (α × SavedState) := do
   if let some (_, state) := reusableResult? then
     set state.meta
   let reusableResult? := reusableResult?.map (fun (val, state) => (val, state.core))
-  controlAt CoreM fun runInCoreM =>
-    Core.withRestoreOrSaveFull reusableResult? fun restore =>
-      runInCoreM <| cont (return { core := (← restore), meta := (← get) })
+  let (a, core) ← controlAt CoreM fun runInBase => do
+    Core.withRestoreOrSaveFull reusableResult? <| runInBase act
+  return (a, { core, meta := (← get) })
 
 instance : MonadBacktrack SavedState MetaM where
   saveState      := Meta.saveState
@@ -1900,6 +1900,10 @@ abbrev isDefEqGuarded (t s : Expr) : MetaM Bool :=
 def isDefEqNoConstantApprox (t s : Expr) : MetaM Bool :=
   approxDefEq <| isDefEq t s
 
+/-- Shorthand for `isDefEq (mkMVar mvarId) val` -/
+def _root_.Lean.MVarId.checkedAssign (mvarId : MVarId) (val : Expr) : MetaM Bool :=
+  isDefEq (mkMVar mvarId) val
+
 /--
   Eta expand the given expression.
   Example:

src/Lean/Meta/DecLevel.lean

@@ -70,4 +70,7 @@ def decLevel (u : Level) : MetaM Level := do
 def getDecLevel (type : Expr) : MetaM Level := do
   decLevel (← getLevel type)
 
+builtin_initialize
+  registerTraceClass `Meta.isLevelDefEq.step
+
 end Lean.Meta

src/Lean/Meta/ExprDefEq.lean

@@ -929,6 +929,91 @@ partial def check
 
 end CheckAssignmentQuick
 
+/--
+Auxiliary function used at `typeOccursCheckImp`.
+Given `type`, it tries to eliminate "dependencies". For example, suppose we are trying to
+perform the assignment `?m := f (?n a b)` where
+```
+?n : let k := g ?m; A -> h k ?m -> C
+```
+If we just perform occurs check `?m` at the type of `?n`, we get a failure, but
+we claim these occurrences are ok because the type `?n a b : C`.
+In the example above, `typeOccursCheckImp` invokes this function with `n := 2`.
+Note that we avoid using `whnf` and `inferType` at `typeOccursCheckImp` to minimize the
+performance impact of this extra check.
+
+See test `typeOccursCheckIssue.lean` for an example where this refinement is needed.
+The test is derived from a Mathlib file.
+-/
+private partial def skipAtMostNumBinders (type : Expr) (n : Nat) : Expr :=
+  match type, n with
+  | .forallE _ _ b _, n+1 => skipAtMostNumBinders b n
+  | .mdata _ b,       n   => skipAtMostNumBinders b n
+  | .letE _ _ v b _,  n   => skipAtMostNumBinders (b.instantiate1 v) n
+  | type,             _   => type
+
+/-- `typeOccursCheck` implementation using unsafe (i.e., pointer equality) features. -/
+private unsafe def typeOccursCheckImp (mctx : MetavarContext) (mvarId : MVarId) (v : Expr) : Bool :=
+  if v.hasExprMVar then
+    visit v |>.run' mkPtrSet
+  else
+    true
+where
+  alreadyVisited (e : Expr) : StateM (PtrSet Expr) Bool := do
+    if (← get).contains e then
+      return true
+    else
+      modify fun s => s.insert e
+      return false
+  occursCheck (type : Expr) : Bool :=
+    let go : StateM MetavarContext Bool := do
+      Lean.occursCheck mvarId type
+    -- Remark: it is ok to discard the the "updated" `MetavarContext` because
+    -- this function assumes all assigned metavariables have already been
+    -- instantiated.
+    go.run' mctx
+  visitMVar (mvarId' : MVarId) (numArgs : Nat := 0) : Bool :=
+    if let some mvarDecl := mctx.findDecl? mvarId' then
+      occursCheck (skipAtMostNumBinders mvarDecl.type numArgs)
+    else
+      false
+  visitApp (e : Expr) : StateM (PtrSet Expr) Bool :=
+    e.withApp fun f args => do
+      unless (← args.allM visit) do
+        return false
+      if f.isMVar then
+        return visitMVar f.mvarId! args.size
+      else
+        visit f
+  visit (e : Expr) : StateM (PtrSet Expr) Bool := do
+    if !e.hasExprMVar then
+      return true
+    else if (← alreadyVisited e) then
+      return true
+    else match e with
+      | .mdata _ b       => visit b
+      | .proj _ _ s      => visit s
+      | .app ..          => visitApp e
+      | .lam _ d b _     => visit d <&&> visit b
+      | .forallE _ d b _ => visit d <&&> visit b
+      | .letE _ t v b _  => visit t <&&> visit v <&&> visit b
+      | .mvar mvarId'    => return visitMVar mvarId'
+      | .bvar .. | .sort .. | .const .. | .fvar ..
+      | .lit .. => return true
+
+/--
+Check whether there are invalid occurrences of `mvarId` in the type of other metavariables in `v`.
+For example, suppose we have
+```
+?m_1 : Nat
+?m_2 : Fin ?m_1
+```
+The assignment `?m_1 := (?m_2).1` should not be accepted.
+See issue #4405 for additional examples.
+-/
+private def typeOccursCheck (mctx : MetavarContext) (mvarId : MVarId) (v : Expr) : Bool :=
+  unsafe typeOccursCheckImp mctx mvarId v
+
 /--
   Auxiliary function for handling constraints of the form `?m a₁ ... aₙ =?= v`.
   It will check whether we can perform the assignment
@@ -951,11 +1036,15 @@ def checkAssignment (mvarId : MVarId) (fvars : Array Expr) (v : Expr) : MetaM (O
     let hasCtxLocals := fvars.any fun fvar => mvarDecl.lctx.containsFVar fvar
     let ctx ← read
     let mctx ← getMCtx
-    if CheckAssignmentQuick.check hasCtxLocals mctx ctx.lctx mvarDecl mvarId fvars v then
-      pure (some v)
+    let v ← if CheckAssignmentQuick.check hasCtxLocals mctx ctx.lctx mvarDecl mvarId fvars v then
+      pure v
+    else if let some v ← CheckAssignment.checkAssignmentAux mvarId fvars hasCtxLocals (← instantiateMVars v) then
+      pure v
     else
-      let v ← instantiateMVars v
-      CheckAssignment.checkAssignmentAux mvarId fvars hasCtxLocals v
+      return none
+    unless typeOccursCheck (← getMCtx) mvarId v do
+      return none
+    return some v
 
 private def processAssignmentFOApproxAux (mvar : Expr) (args : Array Expr) (v : Expr) : MetaM Bool :=
   match v with

src/Lean/Meta/LazyDiscrTree.lean

@@ -801,23 +801,10 @@ structure Cache where
 
 def Cache.empty (ngen : NameGenerator) : Cache := { ngen := ngen, core := {}, meta := {} }
 
-def matchPrefix (s : String) (pre : String) :=
-  s.startsWith pre && (s |>.drop pre.length |>.all Char.isDigit)
-
-def isInternalDetail : Name → Bool
-  | .str p s     =>
-    s.startsWith "_"
-      || matchPrefix s "eq_"
-      || matchPrefix s "match_"
-      || matchPrefix s "proof_"
-      || p.isInternalOrNum
-  | .num _ _     => true
-  | p            => p.isInternalOrNum
-
 def blacklistInsertion (env : Environment) (declName : Name) : Bool :=
   !allowCompletion env declName
   || declName == ``sorryAx
-  || isInternalDetail declName
+  || declName.isInternalDetail
   || (declName matches .str _ "inj")
   || (declName matches .str _ "noConfusionType")
 

src/Lean/Meta/LitValues.lean

@@ -69,15 +69,27 @@ def getFinValue? (e : Expr) : MetaM (Option ((n : Nat) × Fin n)) := OptionT.run
   | 0 => failure
   | m+1 => return ⟨m+1, Fin.ofNat v⟩
 
-/-- Return `some ⟨n, v⟩` if `e` is af `OfNat.ofNat` application encoding a `BitVec n` with value `v` -/
+/--
+Return `some ⟨n, v⟩` if `e` is:
+- an `OfNat.ofNat` application
+- a `BitVec.ofNat` application
+- a `BitVec.ofNatLt` application
+that encode a `BitVec n` with value `v`.
+-/
 def getBitVecValue? (e : Expr) : MetaM (Option ((n : Nat) × BitVec n)) := OptionT.run do
-  if e.isAppOfArity' ``BitVec.ofNat 2 then
-    let n ← getNatValue? (e.getArg!' 0)
-    let v ← getNatValue? (e.getArg!' 1)
+  match_expr e with
+  | BitVec.ofNat nExpr vExpr =>
+    let n ← getNatValue? nExpr
+    let v ← getNatValue? vExpr
+    return ⟨n, BitVec.ofNat n v⟩
+  | BitVec.ofNatLt nExpr vExpr _ =>
+    let n ← getNatValue? nExpr
+    let v ← getNatValue? vExpr
+    return ⟨n, BitVec.ofNat n v⟩
+  | _ =>
+    let (v, type) ← getOfNatValue? e ``BitVec
+    let n ← getNatValue? (← whnfD type.appArg!)
     return ⟨n, BitVec.ofNat n v⟩
-  let (v, type) ← getOfNatValue? e ``BitVec
-  let n ← getNatValue? (← whnfD type.appArg!)
-  return ⟨n, BitVec.ofNat n v⟩
 
 /-- Return `some n` if `e` is an `OfNat.ofNat`-application encoding the `UInt8` with value `n`. -/
 def getUInt8Value? (e : Expr) : MetaM (Option UInt8) := OptionT.run do

src/Lean/Meta/SizeOf.lean

@@ -514,5 +514,9 @@ def mkSizeOfInstances (typeName : Name) : MetaM Unit := do
 
 builtin_initialize
   registerTraceClass `Meta.sizeOf
+  registerTraceClass `Meta.sizeOf.minor
+  registerTraceClass `Meta.sizeOf.minor.step
+  registerTraceClass `Meta.sizeOf.aux
+  registerTraceClass `Meta.sizeOf.loop
 
 end Lean.Meta

src/Lean/Meta/Tactic/AC/Main.lean

@@ -101,11 +101,10 @@ def buildNormProof (preContext : PreContext) (l r : Expr) : MetaM (Lean.Expr ×
 where
   mkContext (α : Expr) (u : Level) (vars : Array Expr) : MetaM (Array Bool × Expr) := do
     let arbitrary := vars[0]!
-    let zero := mkLevelZeroEx ()
-    let plift := mkApp (mkConst ``PLift [zero])
-    let pliftUp := mkApp2 (mkConst ``PLift.up [zero])
-    let noneE tp   := mkApp  (mkConst ``Option.none [zero]) (plift tp)
-    let someE tp v := mkApp2 (mkConst ``Option.some [zero]) (plift tp) (pliftUp tp v)
+    let plift := mkApp (mkConst ``PLift [.zero])
+    let pliftUp := mkApp2 (mkConst ``PLift.up [.zero])
+    let noneE tp   := mkApp  (mkConst ``Option.none [.zero]) (plift tp)
+    let someE tp v := mkApp2 (mkConst ``Option.some [.zero]) (plift tp) (pliftUp tp v)
     let vars ← vars.mapM fun x => do
       let isNeutral :=
         let isNeutralClass := mkApp3 (mkConst ``LawfulIdentity [u]) α preContext.op x