chore: minimization of variable-induced slowdown

We must erase macro scopes for tags in `case` as we do in `cases .. with
..` and `induction .. with ..`.

RELEASES.md

@@ -8,6 +8,7 @@ This file contains work-in-progress notes for the upcoming release, as well as p
 Please check the [releases](https://github.com/leanprover/lean4/releases) page for the current status
 of each version.
 
+v4.8.0
 v4.9.0 (development in progress)
 ---------
 
@@ -22,6 +23,18 @@ v4.9.0 (development in progress)
   definition itself can be marked as `@[semireducible]` to get the previous
   behavor.
 
+* The `MessageData.ofPPFormat` constructor has been removed.
+  Its functionality has been split into two:
+
+  - for lazy structured messages, please use `MessageData.lazy`;
+  - for embedding `Format` or `FormatWithInfos`, use `MessageData.ofFormatWithInfos`.
+
+  An example migration can be found in [#3929](https://github.com/leanprover/lean4/pull/3929/files#diff-5910592ab7452a0e1b2616c62d22202d2291a9ebb463145f198685aed6299867L109).
+
+* The `MessageData.ofFormat` constructor has been turned into a function.
+  If you need to inspect `MessageData`,
+  you can pattern-match on `MessageData.ofFormatWithInfos`.
+
 v4.8.0
 ---------
 
@@ -119,8 +132,462 @@ v4.8.0
 * The `#guard_msgs` command now supports showing a diff between the expected and actual outputs. This feature is currently
   disabled by default, but can be enabled with `set_option guard_msgs.diff true`. Depending on user feedback, this option
   may default to `true` in a future version of Lean.
+### Language features, tactics, and metaprograms
 
-Breaking changes:
+* **Functional induction principles.**
+  [#3432](https://github.com/leanprover/lean4/pull/3432), [#3620](https://github.com/leanprover/lean4/pull/3620),
+  [#3754](https://github.com/leanprover/lean4/pull/3754), [#3762](https://github.com/leanprover/lean4/pull/3762),
+  [#3738](https://github.com/leanprover/lean4/pull/3738), [#3776](https://github.com/leanprover/lean4/pull/3776),
+  [#3898](https://github.com/leanprover/lean4/pull/3898).
+
+  Derived from the definition of a (possibly mutually) recursive function,
+  a **functional induction principle** is created that is tailored to proofs about that function.
+
+  For example from:
+  ```
+  def ackermann : Nat → Nat → Nat
+    | 0, m => m + 1
+    | n+1, 0 => ackermann n 1
+    | n+1, m+1 => ackermann n (ackermann (n + 1) m)
+  ```
+  we get
+  ```
+  ackermann.induct (motive : Nat → Nat → Prop) (case1 : ∀ (m : Nat), motive 0 m)
+    (case2 : ∀ (n : Nat), motive n 1 → motive (Nat.succ n) 0)
+    (case3 : ∀ (n m : Nat), motive (n + 1) m → motive n (ackermann (n + 1) m) → motive (Nat.succ n) (Nat.succ m))
+    (x x : Nat) : motive x x
+  ```
+
+  It can be used in the `induction` tactic using the `using` syntax:
+  ```
+  induction n, m using ackermann.induct
+  ```
+* The termination checker now recognizes more recursion patterns without an
+  explicit `termination_by`. In particular the idiom of counting up to an upper
+  bound, as in
+  ```
+  def Array.sum (arr : Array Nat) (i acc : Nat) : Nat :=
+    if _ : i < arr.size then
+      Array.sum arr (i+1) (acc + arr[i])
+    else
+      acc
+  ```
+  is recognized without having to say `termination_by arr.size - i`.
+  * [#3630](https://github.com/leanprover/lean4/pull/3630) makes `termination_by?` not use `sizeOf` when not needed
+  * [#3652](https://github.com/leanprover/lean4/pull/3652) improves the `termination_by` syntax.
+  * [#3658](https://github.com/leanprover/lean4/pull/3658) changes how termination arguments are elaborated.
+  * [#3665](https://github.com/leanprover/lean4/pull/3665) refactors GuessLex to allow inferring more complex termination arguments
+  * [#3666](https://github.com/leanprover/lean4/pull/3666) infers termination arguments such as `xs.size - i`
+* [#3629](https://github.com/leanprover/lean4/pull/3629),
+  [#3655](https://github.com/leanprover/lean4/pull/3655),
+  [#3747](https://github.com/leanprover/lean4/pull/3747):
+  Adds `@[induction_eliminator]` and `@[cases_eliminator]` attributes to be able to define custom eliminators
+  for the `induction` and `cases` tactics, replacing the `@[eliminator]` attribute.
+  Gives custom eliminators for `Nat` so that `induction` and `cases` put goal states into terms of `0` and `n + 1`
+  rather than `Nat.zero` and `Nat.succ n`.
+  Added option `tactic.customEliminators` to control whether to use custom eliminators.
+  Added a hack for `rcases`/`rintro`/`obtain` to use the custom eliminator for `Nat`.
+* **Shorter instances names.** There is a new algorithm for generating names for anonymous instances.
+  Across Std and Mathlib, the median ratio between lengths of new names and of old names is about 72%.
+  With the old algorithm, the longest name was 1660 characters, and now the longest name is 202 characters.
+  The new algorithm's 95th percentile name length is 67 characters, versus 278 for the old algorithm.
+  While the new algorithm produces names that are 1.2% less unique,
+  it avoids cross-project collisions by adding a module-based suffix
+  when it does not refer to declarations from the same "project" (modules that share the same root).
+  [#3089](https://github.com/leanprover/lean4/pull/3089)
+  and [#3934](https://github.com/leanprover/lean4/pull/3934).
+* [8d2adf](https://github.com/leanprover/lean4/commit/8d2adf521d2b7636347a5b01bfe473bf0fcfaf31)
+  Importing two different files containing proofs of the same theorem is no longer considered an error.
+  This feature is particularly useful for theorems that are automatically generated on demand (e.g., equational theorems).
+* [84b091](https://github.com/leanprover/lean4/commit/84b0919a116e9be12f933e764474f45d964ce85c)
+  Lean now generates an error if the type of a theorem is **not** a proposition.
+* **Definition transparency.** [47a343](https://github.com/leanprover/lean4/commit/47a34316fc03ce936fddd2d3dce44784c5bcdfa9). `@[reducible]`, `@[semireducible]`, and `@[irreducible]` are now scoped and able to be set for imported declarations.
+* `simp`/`dsimp`
+  * [#3607](https://github.com/leanprover/lean4/pull/3607) enables kernel projection reduction in `dsimp`
+  * [b24fbf](https://github.com/leanprover/lean4/commit/b24fbf44f3aaa112f5d799ef2a341772d1eb222d)
+    and [acdb00](https://github.com/leanprover/lean4/commit/acdb0054d5a0efa724cff596ac26852fad5724c4):
+    `dsimproc` command
+    to define defeq-preserving simplification procedures.
+  * [#3624](https://github.com/leanprover/lean4/pull/3624) makes `dsimp` normalize raw nat literals as `OfNat.ofNat` applications.
+  * [#3628](https://github.com/leanprover/lean4/pull/3628) makes `simp` correctly handle `OfScientific.ofScientific` literals.
+  * [#3654](https://github.com/leanprover/lean4/pull/3654) makes `dsimp?` report used simprocs.
+  * [dee074](https://github.com/leanprover/lean4/commit/dee074dcde03a37b7895a4901df2e4fa490c73c7) fixes equation theorem
+    handling in `simp` for non-recursive definitions.
+  * [#3819](https://github.com/leanprover/lean4/pull/3819) improved performance when simp encounters a loop.
+  * [#3821](https://github.com/leanprover/lean4/pull/3821) fixes discharger/cache interaction.
+  * [#3824](https://github.com/leanprover/lean4/pull/3824) keeps `simp` from breaking `Char` literals.
+  * [#3838](https://github.com/leanprover/lean4/pull/3838) allows `Nat` instances matching to be more lenient.
+  * [#3870](https://github.com/leanprover/lean4/pull/3870) documentation for `simp` configuration options.
+  * [#3972](https://github.com/leanprover/lean4/pull/3972) fixes simp caching.
+  * [#4044](https://github.com/leanprover/lean4/pull/4044) improves cache behavior for "well-behaved" dischargers.
+* `omega`
+  * [#3639](https://github.com/leanprover/lean4/pull/3639), [#3766](https://github.com/leanprover/lean4/pull/3766),
+    [#3853](https://github.com/leanprover/lean4/pull/3853), [#3875](https://github.com/leanprover/lean4/pull/3875):
+    introduces a term canonicalizer.
+  * [#3736](https://github.com/leanprover/lean4/pull/3736) improves handling of positivity for the modulo operator for `Int`.
+  * [#3828](https://github.com/leanprover/lean4/pull/3828) makes it work as a `simp` discharger.
+  * [#3847](https://github.com/leanprover/lean4/pull/3847) adds helpful error messages.
+* `rfl`
+  * [#3671](https://github.com/leanprover/lean4/pull/3671), [#3708](https://github.com/leanprover/lean4/pull/3708): upstreams the `@[refl]` attribute and the `rfl` tactic.
+  * [#3751](https://github.com/leanprover/lean4/pull/3751) makes `apply_rfl` not operate on `Eq` itself.
+  * [#4067](https://github.com/leanprover/lean4/pull/4067) improves error message when there are no goals.
+* [#3719](https://github.com/leanprover/lean4/pull/3719) upstreams the `rw?` tactic, with fixes and improvements in
+  [#3783](https://github.com/leanprover/lean4/pull/3783), [#3794](https://github.com/leanprover/lean4/pull/3794),
+  [#3911](https://github.com/leanprover/lean4/pull/3911).
+* `conv`
+  * [#3659](https://github.com/leanprover/lean4/pull/3659) adds a `conv` version of the `calc` tactic.
+  * [#3763](https://github.com/leanprover/lean4/pull/3763) makes `conv` clean up using `try with_reducible rfl` instead of `try rfl`.
+* `#guard_msgs`
+  * [#3617](https://github.com/leanprover/lean4/pull/3617) introduces whitespace protection using the `⏎` character.
+  * [#3883](https://github.com/leanprover/lean4/pull/3883):
+    The `#guard_msgs` command now has options to change whitespace normalization and sensitivity to message ordering.
+    For example, `#guard_msgs (whitespace := lax) in cmd` collapses whitespace before checking messages,
+    and `#guard_msgs (ordering := sorted) in cmd` sorts the messages in lexicographic order before checking.
+  * [#3931](https://github.com/leanprover/lean4/pull/3931) adds an unused variables ignore function for `#guard_msgs`.
+  * [#3912](https://github.com/leanprover/lean4/pull/3912) adds a diff between the expected and actual outputs. This feature is currently
+    disabled by default, but can be enabled with `set_option guard_msgs.diff true`.
+    Depending on user feedback, this option may default to `true` in a future version of Lean.
+* `do` **notation**
+  * [#3820](https://github.com/leanprover/lean4/pull/3820) makes it an error to lift `(<- ...)` out of a pure `if ... then ... else ...`
+* **Lazy discrimination trees**
+  * [#3610](https://github.com/leanprover/lean4/pull/3610) fixes a name collision for `LazyDiscrTree` that could lead to cache poisoning.
+  * [#3677](https://github.com/leanprover/lean4/pull/3677) simplifies and fixes `LazyDiscrTree` handling for `exact?`/`apply?`.
+  * [#3685](https://github.com/leanprover/lean4/pull/3685) moves general `exact?`/`apply?` functionality into `LazyDiscrTree`.
+  * [#3769](https://github.com/leanprover/lean4/pull/3769) has lemma selection improvements for `rw?` and `LazyDiscrTree`.
+  * [#3818](https://github.com/leanprover/lean4/pull/3818) improves ordering of matches.
+* [#3590](https://github.com/leanprover/lean4/pull/3590) adds `inductive.autoPromoteIndices` option to be able to disable auto promotion of indices in the `inductive` command.
+* **Miscellaneous bug fixes and improvements**
+  * [#3606](https://github.com/leanprover/lean4/pull/3606) preserves `cache` and `dischargeDepth` fields in `Lean.Meta.Simp.Result.mkEqSymm`.
+  * [#3633](https://github.com/leanprover/lean4/pull/3633) makes `elabTermEnsuringType` respect `errToSorry`, improving error recovery of the `have` tactic.
+  * [#3647](https://github.com/leanprover/lean4/pull/3647) enables `noncomputable unsafe` definitions, for deferring implementations until later.
+  * [#3672](https://github.com/leanprover/lean4/pull/3672) adjust namespaces of tactics.
+  * [#3725](https://github.com/leanprover/lean4/pull/3725) fixes `Ord` derive handler for indexed inductive types with unused alternatives.
+  * [#3893](https://github.com/leanprover/lean4/pull/3893) improves performance of derived `Ord` instances.
+  * [#3771](https://github.com/leanprover/lean4/pull/3771) changes error reporting for failing tactic macros. Improves `rfl` error message.
+  * [#3745](https://github.com/leanprover/lean4/pull/3745) fixes elaboration of generalized field notation if the object of the notation is an optional parameter.
+  * [#3799](https://github.com/leanprover/lean4/pull/3799) makes commands such as `universe`, `variable`, `namespace`, etc. require that their argument appear in a later column.
+    Commands that can optionally parse an `ident` or parse any number of `ident`s generally should require
+    that the `ident` use `colGt`. This keeps typos in commands from being interpreted as identifiers.
+  * [#3815](https://github.com/leanprover/lean4/pull/3815) lets the `split` tactic be used for writing code.
+  * [#3822](https://github.com/leanprover/lean4/pull/3822) adds missing info in `induction` tactic for `with` clauses of the form `| cstr a b c => ?_`.
+  * [#3806](https://github.com/leanprover/lean4/pull/3806) fixes `withSetOptionIn` combinator.
+  * [#3844](https://github.com/leanprover/lean4/pull/3844) removes unused `trace.Elab.syntax` option.
+  * [#3896](https://github.com/leanprover/lean4/pull/3896) improves hover and go-to-def for `attribute` command.
+  * [#3989](https://github.com/leanprover/lean4/pull/3989) makes linter options more discoverable.
+  * [#3916](https://github.com/leanprover/lean4/pull/3916) fixes go-to-def for syntax defined with `@[builtin_term_parser]`.
+  * [#3962](https://github.com/leanprover/lean4/pull/3962) fixes how `solveByElim` handles `symm` lemmas, making `exact?`/`apply?` usable again.
+  * [#3968](https://github.com/leanprover/lean4/pull/3968) improves the `@[deprecated]` attribute, adding `(since := "<date>")` field.
+  * [#3768](https://github.com/leanprover/lean4/pull/3768) makes `#print` command show structure fields.
+  * [#3974](https://github.com/leanprover/lean4/pull/3974) makes `exact?%` behave like `by exact?` rather than `by apply?`.
+  * [#3994](https://github.com/leanprover/lean4/pull/3994) makes elaboration of `he ▸ h` notation more predictable.
+  * [#3991](https://github.com/leanprover/lean4/pull/3991) adjusts transparency for `decreasing_trivial` macros.
+  * [#4092](https://github.com/leanprover/lean4/pull/4092) improves performance of `binop%` and `binrel%` expression tree elaborators.
+* **Docs:** [#3748](https://github.com/leanprover/lean4/pull/3748), [#3796](https://github.com/leanprover/lean4/pull/3796),
+[#3800](https://github.com/leanprover/lean4/pull/3800), [#3874](https://github.com/leanprover/lean4/pull/3874),
+[#3863](https://github.com/leanprover/lean4/pull/3863), [#3862](https://github.com/leanprover/lean4/pull/3862),
+[#3891](https://github.com/leanprover/lean4/pull/3891), [#3873](https://github.com/leanprover/lean4/pull/3873),
+[#3908](https://github.com/leanprover/lean4/pull/3908), [#3872](https://github.com/leanprover/lean4/pull/3872).
+
+### Language server and IDE extensions
+
+* [#3432](https://github.com/leanprover/lean4/pull/3432) enables `import` auto-completions.
+* [#3608](https://github.com/leanprover/lean4/pull/3608) fixes issue [leanprover/vscode-lean4#392](https://github.com/leanprover/vscode-lean4/issues/392).
+  Diagnostic ranges had an off-by-one error that would misplace goal states for example.
+* [#3014](https://github.com/leanprover/lean4/pull/3014) introduces snapshot trees, foundational work for incremental tactics and parallelism.
+  [#3849](https://github.com/leanprover/lean4/pull/3849) adds basic incrementality API.
+* [#3271](https://github.com/leanprover/lean4/pull/3271) adds support for server-to-client requests.
+* [#3656](https://github.com/leanprover/lean4/pull/3656) fixes jump to definition when there are conflicting names from different files.
+  Fixes issue [#1170](https://github.com/leanprover/lean4/issues/1170).
+* [#3691](https://github.com/leanprover/lean4/pull/3691), [#3925](https://github.com/leanprover/lean4/pull/3925),
+  [#3932](https://github.com/leanprover/lean4/pull/3932) keep semantic tokens synchronized (used for semantic highlighting), with performance improvements.
+* [#3247](https://github.com/leanprover/lean4/pull/3247) and [#3730](https://github.com/leanprover/lean4/pull/3730)
+  add diagnostics to run "Restart File" when a file dependency is saved.
+* [#3722](https://github.com/leanprover/lean4/pull/3722) uses the correct module names when displaying references.
+* [#3728](https://github.com/leanprover/lean4/pull/3728) makes errors in header reliably appear and makes the "Import out of date" warning be at "hint" severity.
+  [#3739](https://github.com/leanprover/lean4/pull/3739) simplifies the text of this warning.
+* [#3778](https://github.com/leanprover/lean4/pull/3778) fixes [#3462](https://github.com/leanprover/lean4/issues/3462),
+  where info nodes from before the cursor would be used for computing completions.
+* [#3985](https://github.com/leanprover/lean4/pull/3985) makes trace timings appear in Infoview.
+
+### Pretty printing
+
+* [#3797](https://github.com/leanprover/lean4/pull/3797) fixes the hovers over binders so that they show their types.
+* [#3640](https://github.com/leanprover/lean4/pull/3640) and [#3735](https://github.com/leanprover/lean4/pull/3735): Adds attribute `@[pp_using_anonymous_constructor]` to make structures pretty print as `⟨x, y, z⟩`
+  rather than as `{a := x, b := y, c := z}`.
+  This attribute is applied to `Sigma`, `PSigma`, `PProd`, `Subtype`, `And`, and `Fin`.
+* [#3749](https://github.com/leanprover/lean4/pull/3749)
+  Now structure instances pretty print with parent structures' fields inlined.
+  That is, if `B` extends `A`, then `{ toA := { x := 1 }, y := 2 }` now pretty prints as `{ x := 1, y := 2 }`.
+  Setting option `pp.structureInstances.flatten` to false turns this off.
+* [#3737](https://github.com/leanprover/lean4/pull/3737), [#3744](https://github.com/leanprover/lean4/pull/3744)
+  and [#3750](https://github.com/leanprover/lean4/pull/3750):
+  Option `pp.structureProjections` is renamed to `pp.fieldNotation`, and there is now a suboption `pp.fieldNotation.generalized`
+  to enable pretty printing function applications using generalized field notation (defaults to true).
+  Field notation can be disabled on a function-by-function basis using the `@[pp_nodot]` attribute.
+  The notation is not used for theorems.
+* [#4071](https://github.com/leanprover/lean4/pull/4071) fixes interaction between app unexpanders and `pp.fieldNotation.generalized`
+* [#3625](https://github.com/leanprover/lean4/pull/3625) makes `delabConstWithSignature` (used by `#check`) have the ability to put arguments "after the colon"
+  to avoid printing inaccessible names.
+* [#3798](https://github.com/leanprover/lean4/pull/3798),
+  [#3978](https://github.com/leanprover/lean4/pull/3978),
+  [#3798](https://github.com/leanprover/lean4/pull/3980):
+  Adds options `pp.mvars` (default: true) and `pp.mvars.withType` (default: false).
+  When `pp.mvars` is false, expression metavariables pretty print as `?_` and universe metavariables pretty print as `_`.
+  When `pp.mvars.withType` is true, expression metavariables pretty print with a type ascription.
+  These can be set when using `#guard_msgs` to make tests not depend on the particular names of metavariables.
+* [#3917](https://github.com/leanprover/lean4/pull/3917) makes binders hoverable and gives them docstrings.
+* [#4034](https://github.com/leanprover/lean4/pull/4034) makes hovers for RHS terms in `match` expressions in the Infoview reliably show the correct term.
+
+### Library
+
+* `Bool`/`Prop`
+  * [#3508](https://github.com/leanprover/lean4/pull/3508) improves `simp` confluence for `Bool` and `Prop` terms.
+  * Theorems: [#3604](https://github.com/leanprover/lean4/pull/3604)
+* `Nat`
+  * [#3579](https://github.com/leanprover/lean4/pull/3579) makes `Nat.succ_eq_add_one` be a simp lemma, now that `induction`/`cases` uses `n + 1` instead of `Nat.succ n`.
+  * [#3808](https://github.com/leanprover/lean4/pull/3808) replaces `Nat.succ` simp rules with simprocs.
+  * [#3876](https://github.com/leanprover/lean4/pull/3876) adds faster `Nat.repr` implementation in C.
+* `Int`
+  * Theorems: [#3890](https://github.com/leanprover/lean4/pull/3890)
+* `UInt`s
+  * [#3960](https://github.com/leanprover/lean4/pull/3960) improves performance of upcasting.
+* `Array` and `Subarray`
+  * [#3676](https://github.com/leanprover/lean4/pull/3676) removes `Array.eraseIdxAux`, `Array.eraseIdxSzAux`, and `Array.eraseIdx'`.
+  * [#3648](https://github.com/leanprover/lean4/pull/3648) simplifies `Array.findIdx?`.
+  * [#3851](https://github.com/leanprover/lean4/pull/3851) renames fields of `Subarray`.
+* `List`
+  * [#3785](https://github.com/leanprover/lean4/pull/3785) upstreams tail-recursive List operations and `@[csimp]` lemmas.
+* `BitVec`
+  * Theorems: [#3593](https://github.com/leanprover/lean4/pull/3593),
+  [#3593](https://github.com/leanprover/lean4/pull/3593), [#3597](https://github.com/leanprover/lean4/pull/3597),
+  [#3598](https://github.com/leanprover/lean4/pull/3598), [#3721](https://github.com/leanprover/lean4/pull/3721),
+  [#3729](https://github.com/leanprover/lean4/pull/3729), [#3880](https://github.com/leanprover/lean4/pull/3880),
+  [#4039](https://github.com/leanprover/lean4/pull/4039).
+  * [#3884](https://github.com/leanprover/lean4/pull/3884) protects `Std.BitVec`.
+* `String`
+  * [#3832](https://github.com/leanprover/lean4/pull/3832) fixes `String.splitOn`.
+  * [#3959](https://github.com/leanprover/lean4/pull/3959) adds `String.Pos.isValid`.
+  * [#3959](https://github.com/leanprover/lean4/pull/3959) UTF-8 string validation.
+  * [#3961](https://github.com/leanprover/lean4/pull/3961) adds a model implementation for UTF-8 encoding and decoding.
+* `IO`
+  * [#4097](https://github.com/leanprover/lean4/pull/4097) adds `IO.getTaskState` which returns whether a task is finished, actively running, or waiting on other Tasks to finish.
+
+* **Refactors**
+  * [#3605](https://github.com/leanprover/lean4/pull/3605) reduces imports for `Init.Data.Nat` and `Init.Data.Int`.
+  * [#3613](https://github.com/leanprover/lean4/pull/3613) reduces imports for `Init.Omega.Int`.
+  * [#3634](https://github.com/leanprover/lean4/pull/3634) upstreams `Std.Data.Nat`
+    and [#3635](https://github.com/leanprover/lean4/pull/3635) upstreams `Std.Data.Int`.
+  * [#3790](https://github.com/leanprover/lean4/pull/3790) reduces more imports for `omega`.
+  * [#3694](https://github.com/leanprover/lean4/pull/3694) extends `GetElem` interface with `getElem!` and `getElem?` to simplify containers like `RBMap`.
+  * [#3865](https://github.com/leanprover/lean4/pull/3865) renames `Option.toMonad` (see breaking changes below).
+  * [#3882](https://github.com/leanprover/lean4/pull/3882) unifies `lexOrd` with `compareLex`.
+* **Other fixes or improvements**
+  * [#3765](https://github.com/leanprover/lean4/pull/3765) makes `Quotient.sound` be a `theorem`.
+  * [#3645](https://github.com/leanprover/lean4/pull/3645) fixes `System.FilePath.parent` in the case of absolute paths.
+  * [#3660](https://github.com/leanprover/lean4/pull/3660) `ByteArray.toUInt64LE!` and `ByteArray.toUInt64BE!` were swapped.
+  * [#3881](https://github.com/leanprover/lean4/pull/3881), [#3887](https://github.com/leanprover/lean4/pull/3887) fix linearity issues in `HashMap.insertIfNew`, `HashSet.erase`, and `HashMap.erase`.
+    The `HashMap.insertIfNew` fix improves `import` performance.
+  * [#3830](https://github.com/leanprover/lean4/pull/3830) ensures linearity in `Parsec.many*Core`.
+  * [#3930](https://github.com/leanprover/lean4/pull/3930) adds `FS.Stream.isTty` field.
+  * [#3866](https://github.com/leanprover/lean4/pull/3866) deprecates `Option.toBool` in favor of `Option.isSome`.
+  * [#3975](https://github.com/leanprover/lean4/pull/3975) upstreams `Data.List.Init` and `Data.Array.Init` material from Std.
+  * [#3942](https://github.com/leanprover/lean4/pull/3942) adds instances that make `ac_rfl` work without Mathlib.
+  * [#4010](https://github.com/leanprover/lean4/pull/4010) changes `Fin.induction` to use structural induction.
+  * [02753f](https://github.com/leanprover/lean4/commit/02753f6e4c510c385efcbf71fa9a6bec50fce9ab)
+    fixes bug in `reduceLeDiff` simproc.
+  * [#4097](https://github.com/leanprover/lean4/pull/4097)
+    adds `IO.TaskState` and `IO.getTaskState` to get the task from the Lean runtime's task manager.
+* **Docs:** [#3615](https://github.com/leanprover/lean4/pull/3615), [#3664](https://github.com/leanprover/lean4/pull/3664),
+  [#3707](https://github.com/leanprover/lean4/pull/3707), [#3734](https://github.com/leanprover/lean4/pull/3734),
+  [#3868](https://github.com/leanprover/lean4/pull/3868), [#3861](https://github.com/leanprover/lean4/pull/3861),
+  [#3869](https://github.com/leanprover/lean4/pull/3869), [#3858](https://github.com/leanprover/lean4/pull/3858),
+  [#3856](https://github.com/leanprover/lean4/pull/3856), [#3857](https://github.com/leanprover/lean4/pull/3857),
+  [#3867](https://github.com/leanprover/lean4/pull/3867), [#3864](https://github.com/leanprover/lean4/pull/3864),
+  [#3860](https://github.com/leanprover/lean4/pull/3860), [#3859](https://github.com/leanprover/lean4/pull/3859),
+  [#3871](https://github.com/leanprover/lean4/pull/3871), [#3919](https://github.com/leanprover/lean4/pull/3919).
+
+### Lean internals
+
+* **Defeq and WHNF algorithms**
+  * [#3616](https://github.com/leanprover/lean4/pull/3616) gives better support for reducing `Nat.rec` expressions.
+  * [#3774](https://github.com/leanprover/lean4/pull/3774) add tracing for "non-easy" WHNF cases.
+  * [#3807](https://github.com/leanprover/lean4/pull/3807) fixes an `isDefEq` performance issue, now trying structure eta *after* lazy delta reduction.
+  * [#3816](https://github.com/leanprover/lean4/pull/3816) fixes `.yesWithDeltaI` behavior to prevent increasing transparency level when reducing projections.
+  * [#3837](https://github.com/leanprover/lean4/pull/3837) improves heuristic at `isDefEq`.
+  * [#3965](https://github.com/leanprover/lean4/pull/3965) improves `isDefEq` for constraints of the form `t.i =?= s.i`.
+  * [#3977](https://github.com/leanprover/lean4/pull/3977) improves `isDefEqProj`.
+  * [#3981](https://github.com/leanprover/lean4/pull/3981) adds universe constraint approximations to be able to solve `u =?= max u ?v` using `?v = u`.
+    These approximations are only applied when universe constraints cannot be postponed anymore.
+  * [#4004](https://github.com/leanprover/lean4/pull/4004) improves `isDefEqProj` during typeclass resolution.
+  * [#4012](https://github.com/leanprover/lean4/pull/4012) adds `backward.isDefEq.lazyProjDelta` and `backward.isDefEq.lazyWhnfCore` backwards compatibility flags.
+* **Kernel**
+  * [#3966](https://github.com/leanprover/lean4/pull/3966) removes dead code.
+  * [#4035](https://github.com/leanprover/lean4/pull/4035) fixes mismatch for `TheoremVal` between Lean and C++.
+* **Discrimination trees**
+  * [423fed](https://github.com/leanprover/lean4/commit/423fed79a9de75705f34b3e8648db7e076c688d7)
+    and [3218b2](https://github.com/leanprover/lean4/commit/3218b25974d33e92807af3ce42198911c256ff1d):
+    simplify handling of dependent/non-dependent pi types.
+* **Typeclass instance synthesis**
+  * [#3638](https://github.com/leanprover/lean4/pull/3638) eta-reduces synthesized instances
+  * [ce350f](https://github.com/leanprover/lean4/commit/ce350f348161e63fccde6c4a5fe1fd2070e7ce0f) fixes a linearity issue
+  * [917a31](https://github.com/leanprover/lean4/commit/917a31f694f0db44d6907cc2b1485459afe74d49)
+    improves performance by considering at most one answer for subgoals not containing metavariables.
+    [#4008](https://github.com/leanprover/lean4/pull/4008) adds `backward.synthInstance.canonInstances` backward compatibility flag.
+* **Definition processing**
+  * [#3661](https://github.com/leanprover/lean4/pull/3661), [#3767](https://github.com/leanprover/lean4/pull/3767) changes automatically generated equational theorems to be named
+    using suffix `.eq_<idx>` instead of `._eq_<idx>`, and `.eq_def` instead of `._unfold`. (See breaking changes below.)
+    [#3675](https://github.com/leanprover/lean4/pull/3675) adds a mechanism to reserve names.
+    [#3803](https://github.com/leanprover/lean4/pull/3803) fixes reserved name resolution inside namespaces and fixes handling of `match`er declarations and equation lemmas.
+  * [#3662](https://github.com/leanprover/lean4/pull/3662) causes auxiliary definitions nested inside theorems to become `def`s if they are not proofs.
+  * [#4006](https://github.com/leanprover/lean4/pull/4006) makes proposition fields of `structure`s be theorems.
+  * [#4018](https://github.com/leanprover/lean4/pull/4018) makes it an error for a theorem to be `extern`.
+  * [#4047](https://github.com/leanprover/lean4/pull/4047) improves performance making equations for well-founded recursive definitions.
+* **Refactors**
+  * [#3614](https://github.com/leanprover/lean4/pull/3614) avoids unfolding in `Lean.Meta.evalNat`.
+  * [#3621](https://github.com/leanprover/lean4/pull/3621) centralizes functionality for `Fix`/`GuessLex`/`FunInd` in the `ArgsPacker` module.
+  * [#3186](https://github.com/leanprover/lean4/pull/3186) rewrites the UnusedVariable linter to be more performant.
+  * [#3589](https://github.com/leanprover/lean4/pull/3589) removes coercion from `String` to `Name` (see breaking changes below).
+  * [#3237](https://github.com/leanprover/lean4/pull/3237) removes the `lines` field from `FileMap`.
+  * [#3951](https://github.com/leanprover/lean4/pull/3951) makes msg parameter to `throwTacticEx` optional.
+* **Diagnostics**
+  * [#4016](https://github.com/leanprover/lean4/pull/4016), [#4019](https://github.com/leanprover/lean4/pull/4019),
+    [#4020](https://github.com/leanprover/lean4/pull/4020), [#4030](https://github.com/leanprover/lean4/pull/4030),
+    [#4031](https://github.com/leanprover/lean4/pull/4031),
+    [c3714b](https://github.com/leanprover/lean4/commit/c3714bdc6d46845c0428735b283c5b48b23cbcf7),
+    [#4049](https://github.com/leanprover/lean4/pull/4049) adds `set_option diagnostics true` for diagnostic counters.
+    Tracks number of unfolded declarations, instances, reducible declarations, used instances, recursor reductions,
+    `isDefEq` heuristic applications, among others.
+    This option is suggested in exceptional situations, such as at deterministic timeout and maximum recursion depth.
+  * [283587](https://github.com/leanprover/lean4/commit/283587987ab2eb3b56fbc3a19d5f33ab9e04a2ef)
+    adds diagnostic information for `simp`.
+  * [#4043](https://github.com/leanprover/lean4/pull/4043) adds diagnostic information for congruence theorems.
+  * [#4048](https://github.com/leanprover/lean4/pull/4048) display diagnostic information
+    for `set_option diagnostics true in <tactic>` and `set_option diagnostics true in <term>`.
+* **Other features**
+  * [#3800](https://github.com/leanprover/lean4/pull/3800) adds environment extension to record which definitions use structural or well-founded recursion.
+  * [#3801](https://github.com/leanprover/lean4/pull/3801) `trace.profiler` can now export to Firefox Profiler.
+  * [#3918](https://github.com/leanprover/lean4/pull/3918), [#3953](https://github.com/leanprover/lean4/pull/3953) adds `@[builtin_doc]` attribute to make docs and location of a declaration available as a builtin.
+  * [#3939](https://github.com/leanprover/lean4/pull/3939) adds the `lean --json` CLI option to print messages as JSON.
+  * [#3075](https://github.com/leanprover/lean4/pull/3075) improves `test_extern` command.
+  * [#3970](https://github.com/leanprover/lean4/pull/3970) gives monadic generalization of `FindExpr`.
+* **Docs:** [#3743](https://github.com/leanprover/lean4/pull/3743), [#3921](https://github.com/leanprover/lean4/pull/3921),
+  [#3954](https://github.com/leanprover/lean4/pull/3954).
+* **Other fixes:** [#3622](https://github.com/leanprover/lean4/pull/3622),
+  [#3726](https://github.com/leanprover/lean4/pull/3726), [#3823](https://github.com/leanprover/lean4/pull/3823),
+  [#3897](https://github.com/leanprover/lean4/pull/3897), [#3964](https://github.com/leanprover/lean4/pull/3964),
+  [#3946](https://github.com/leanprover/lean4/pull/3946), [#4007](https://github.com/leanprover/lean4/pull/4007),
+  [#4026](https://github.com/leanprover/lean4/pull/4026).
+
+### Compiler, runtime, and FFI
+
+* [#3632](https://github.com/leanprover/lean4/pull/3632) makes it possible to allocate and free thread-local runtime resources for threads not started by Lean itself.
+* [#3627](https://github.com/leanprover/lean4/pull/3627) improves error message about compacting closures.
+* [#3692](https://github.com/leanprover/lean4/pull/3692) fixes deadlock in `IO.Promise.resolve`.
+* [#3753](https://github.com/leanprover/lean4/pull/3753) catches error code from `MoveFileEx` on Windows.
+* [#4028](https://github.com/leanprover/lean4/pull/4028) fixes a double `reset` bug in `ResetReuse` transformation.
+* [6e731b](https://github.com/leanprover/lean4/commit/6e731b4370000a8e7a5cfb675a7f3d7635d21f58)
+  removes `interpreter` copy constructor to avoid potential memory safety issues.
+
+### Lake
+
+* **TOML Lake configurations**. [#3298](https://github.com/leanprover/lean4/pull/3298), [#4104](https://github.com/leanprover/lean4/pull/4104).
+
+  Lake packages can now use TOML as a alternative configuration file format instead of Lean. If the default `lakefile.lean` is missing, Lake will also look for a `lakefile.toml`. The TOML version of the configuration supports a restricted set of the Lake configuration options, only including those which can easily mapped to a TOML data structure. The TOML syntax itself fully compiles with the TOML v1.0.0 specification.
+
+  As part of the introduction of this new feature, we have been helping maintainers of some major packages within the ecosystem switch to this format. For example, the following is Aesop's new `lakefile.toml`:
+
+
+  **[leanprover-community/aesop/lakefile.toml](https://raw.githubusercontent.com/leanprover-community/aesop/de11e0ecf372976e6d627c210573146153090d2d/lakefile.toml)**
+  ```toml
+  name = "aesop"
+  defaultTargets = ["Aesop"]
+  testRunner = "test"
+  precompileModules = false
+
+  [[require]]
+  name = "batteries"
+  git = "https://github.com/leanprover-community/batteries"
+  rev = "main"
+
+  [[lean_lib]]
+  name = "Aesop"
+
+  [[lean_lib]]
+  name = "AesopTest"
+  globs = ["AesopTest.+"]
+  leanOptions = {linter.unusedVariables = false}
+
+  [[lean_exe]]
+  name = "test"
+  srcDir = "scripts"
+  ```
+
+  To assist users who wish to transition their packages between configuration file formats, there is also a new `lake translate-config` command for migrating to/from TOML.
+
+  Running `lake translate-config toml` will produce a `lakefile.toml` version of a package's `lakefile.lean`. Any configuration options unsupported by the TOML format will be discarded during translation, but the original `lakefile.lean` will remain so that you can verify the translation looks good before deleting it.
+
+* **Build progress overhaul.** [#3835](https://github.com/leanprover/lean4/pull/3835), [#4115](https://github.com/leanprover/lean4/pull/4115), [#4127](https://github.com/leanprover/lean4/pull/4127), [#4220](https://github.com/leanprover/lean4/pull/4220), [#4232](https://github.com/leanprover/lean4/pull/4232), [#4236](https://github.com/leanprover/lean4/pull/4236).
+
+  Builds are now managed by a top-level Lake build monitor, this makes the output of Lake builds more standardized and enables producing prettier and more configurable progress reports.
+
+  As part of this change, job isolation has improved. Stray I/O and other build related errors in custom targets are now properly isolated and caught as part of their job. Import errors no longer cause Lake to abort the entire build and are instead localized to the build jobs of the modules in question.
+
+  Lake also now uses ANSI escape sequences to add color and produce progress lines that update in-place; this can be toggled on and off using `--ansi` / `--no-ansi`.
+
+
+  `--wfail` and `--iofail` options have been added that causes a build to fail if any of the jobs log a warning (`--wfail`) or produce any output or log information messages (`--iofail`). Unlike some other build systems, these options do **NOT** convert these logs into errors, and Lake does not abort jobs on such a log (i.e., dependent jobs will still continue unimpeded).
+
+* `lake test`. [#3779](https://github.com/leanprover/lean4/pull/3779).
+
+  Lake now has a built-in `test` command which will run a script or executable labelled `@[test_runner]` (in Lean) or defined as the `testRunner` (in TOML) in the root package.
+
+  Lake also provides a `lake check-test` command which will exit with code `0` if the package has a properly configured test runner or error with `1` otherwise.
+
+* `lake lean`. [#3793](https://github.com/leanprover/lean4/pull/3793).
+
+  The new command `lake lean <file> [-- <args...>]` functions like `lake env lean <file> <args...>`, except that it builds the imports of `file` before running `lean`. This makes it very useful for running test or example code that imports modules that are not guaranteed to have been built beforehand.
+
+* **Miscellaneous bug fixes and improvements**
+  * [#3609](https://github.com/leanprover/lean4/pull/3609) `LEAN_GITHASH` environment variable to override the detected Git hash for Lean when computing traces, useful for testing custom builds of Lean.
+  * [#3795](https://github.com/leanprover/lean4/pull/3795) improves relative package directory path normalization in the pre-rename check.
+  * [#3957](https://github.com/leanprover/lean4/pull/3957) fixes handling of packages that appear multiple times in a dependency tree.
+  * [#3999](https://github.com/leanprover/lean4/pull/3999) makes it an error for there to be a mismatch between a package name and what it is required as. Also adds a special message for the `std`-to-`batteries` rename.
+  * [#4033](https://github.com/leanprover/lean4/pull/4033) fixes quiet mode.
+* **Docs:** [#3704](https://github.com/leanprover/lean4/pull/3704).
+
+### DevOps
+
+* [#3536](https://github.com/leanprover/lean4/pull/3536) and [#3833](https://github.com/leanprover/lean4/pull/3833)
+  add a checklist for the release process.
+* [#3600](https://github.com/leanprover/lean4/pull/3600) runs nix-ci more uniformly.
+* [#3612](https://github.com/leanprover/lean4/pull/3612) avoids argument limits when building on Windows.
+* [#3682](https://github.com/leanprover/lean4/pull/3682) builds Lean's `.o` files in parallel to rest of core.
+* [#3601](https://github.com/leanprover/lean4/pull/3601)
+  changes the way Lean is built on Windows (see breaking changes below).
+  As a result, Lake now dynamically links executables with `supportInterpreter := true` on Windows
+  to `libleanshared.dll` and `libInit_shared.dll`. Therefore, such executables will not run
+  unless those shared libraries are co-located with the executables or part of `PATH`.
+  Running the executable via `lake exe` will ensure these libraries are part of `PATH`.
+
+  In a related change, the signature of the `nativeFacets` Lake configuration options has changed
+  from a static `Array` to a function `(shouldExport : Bool) → Array`.
+  See its docstring or Lake's [README](src/lake/README.md) for further details on the changed option.
+* [#3690](https://github.com/leanprover/lean4/pull/3690) marks "Build matrix complete" as canceled if the build is canceled.
+* [#3700](https://github.com/leanprover/lean4/pull/3700), [#3702](https://github.com/leanprover/lean4/pull/3702),
+  [#3701](https://github.com/leanprover/lean4/pull/3701), [#3834](https://github.com/leanprover/lean4/pull/3834),
+  [#3923](https://github.com/leanprover/lean4/pull/3923): fixes and improvements for std and mathlib CI.
+* [#3712](https://github.com/leanprover/lean4/pull/3712) fixes `nix build .` on macOS.
+* [#3717](https://github.com/leanprover/lean4/pull/3717) replaces `shell.nix` in devShell with `flake.nix`.
+* [#3715](https://github.com/leanprover/lean4/pull/3715) and [#3790](https://github.com/leanprover/lean4/pull/3790) add test result summaries.
+* [#3971](https://github.com/leanprover/lean4/pull/3971) prevents stage0 changes via the merge queue.
+* [#3979](https://github.com/leanprover/lean4/pull/3979) adds handling for `changes-stage0` label.
+* [#3952](https://github.com/leanprover/lean4/pull/3952) adds a script to summarize GitHub issues.
+* [18a699](https://github.com/leanprover/lean4/commit/18a69914da53dbe37c91bc2b9ce65e1dc01752b6)
+  fixes asan linking
+
+### Breaking changes
+
+* Due to the major Lake build refactor, code using the affected parts of the Lake API or relying on the previous output format of Lake builds is likely to have been broken. We have tried to minimize the breakages and, were possible, old definitions have been marked `@[deprecated]` with a reference to the new alternative.
 
 * Automatically generated equational theorems are now named using suffix `.eq_<idx>` instead of `._eq_<idx>`, and `.def` instead of `._unfold`. Example:
 ```

doc/examples/test_single.sh

@@ -1,4 +1,4 @@
 #!/usr/bin/env bash
 source ../../tests/common.sh
 
-exec_check lean -j 0 -Dlinter.all=false "$f"
+exec_check lean -Dlinter.all=false "$f"

doc/latex/lean4.py

@@ -1,100 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
-    pygments.lexers.theorem
-    ~~~~~~~~~~~~~~~~~~~~~~~
-
-    Lexers for theorem-proving languages.
-
-    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
-    :license: BSD, see LICENSE for details.
-"""
-
-import re
-
-from pygments.lexer import RegexLexer, default, words
-from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
-    Number, Punctuation, Generic
-
-__all__ = ['Lean4Lexer']
-
-class Lean4Lexer(RegexLexer):
-    """
-    For the `Lean 4 <https://github.com/leanprover/lean4>`_
-    theorem prover.
-
-    .. versionadded:: 2.0
-    """
-    name = 'Lean4'
-    aliases = ['lean4']
-    filenames = ['*.lean']
-    mimetypes = ['text/x-lean']
-
-    flags = re.MULTILINE | re.UNICODE
-
-    keywords1 = (
-        'import', 'abbreviation', 'opaque_hint', 'tactic_hint', 'definition',
-        'renaming', 'inline', 'hiding', 'parameter', 'lemma', 'variable',
-        'theorem', 'axiom', 'inductive', 'structure', 'universe', 'alias',
-        'help', 'options', 'precedence', 'postfix', 'prefix',
-        'infix', 'infixl', 'infixr', 'notation', '#eval',
-        '#check', '#reduce', '#exit', 'coercion', 'end', 'private', 'using', 'namespace',
-        'including', 'instance', 'section', 'context', 'protected', 'expose',
-        'export', 'set_option', 'extends', 'open', 'example',
-        'constant', 'constants', 'print', 'opaque', 'reducible', 'irreducible',
-        'def', 'macro', 'elab', 'syntax', 'macro_rules', 'reduce', 'where',
-        'abbrev', 'noncomputable', 'class', 'attribute', 'synth', 'mutual',
-    )
-
-    keywords2 = (
-        'forall', 'fun', 'Pi', 'obtain', 'from', 'have', 'show', 'assume',
-        'take', 'let', 'if', 'else', 'then', 'by', 'in', 'with', 'begin',
-        'proof', 'qed', 'calc', 'match', 'nomatch', 'do', 'at',
-    )
-
-    keywords3 = (
-        # Sorts
-        'Type', 'Prop', 'Sort',
-    )
-
-    operators = (
-        u'!=', u'#', u'&', u'&&', u'*', u'+', u'-', u'/', u'@', u'!', u'`',
-        u'-.', u'->', u'.', u'..', u'...', u'::', u':>', u';', u';;', u'<',
-        u'<-', u'=', u'==', u'>', u'_', u'|', u'||', u'~', u'=>', u'<=', u'>=',
-        u'/\\', u'\\/', u'∀', u'Π', u'λ', u'↔', u'∧', u'∨', u'≠', u'≤', u'≥',
-        u'¬', u'⁻¹', u'⬝', u'▸', u'→', u'∃', u'ℕ', u'ℤ', u'≈', u'×', u'⌞',
-        u'⌟', u'≡', u'⟨', u'⟩',
-    )
-
-    punctuation = (u'(', u')', u':', u'{', u'}', u'[', u']', u'⦃', u'⦄',
-                   u':=', u',')
-
-    tokens = {
-        'root': [
-            (r'\s+', Text),
-            (r'/-', Comment, 'comment'),
-            (r'--.*?$', Comment.Single),
-            (words(keywords1, prefix=r'\b', suffix=r'\b'), Keyword.Namespace),
-            (words(keywords2, prefix=r'\b', suffix=r'\b'), Keyword),
-            (words(keywords3, prefix=r'\b', suffix=r'\b'), Keyword.Type),
-            (words(operators), Name.Builtin.Pseudo),
-            (words(punctuation), Operator),
-            (u"[A-Za-z_\u03b1-\u03ba\u03bc-\u03fb\u1f00-\u1ffe\u2100-\u214f]"
-             u"[A-Za-z_'\u03b1-\u03ba\u03bc-\u03fb\u1f00-\u1ffe\u2070-\u2079"
-             u"\u207f-\u2089\u2090-\u209c\u2100-\u214f0-9]*", Name),
-            (r'\d+', Number.Integer),
-            (r'"', String.Double, 'string'),
-            (r'[~?][a-z][\w\']*:', Name.Variable)
-        ],
-        'comment': [
-            # Multiline Comments
-            (r'[^/-]', Comment.Multiline),
-            (r'/-', Comment.Multiline, '#push'),
-            (r'-/', Comment.Multiline, '#pop'),
-            (r'[/-]', Comment.Multiline)
-        ],
-        'string': [
-            (r'[^\\"]+', String.Double),
-            (r'\\[n"\\]', String.Escape),
-            ('"', String.Double, '#pop'),
-        ],
-    }

doc/syntax_highlight_in_latex.md

@@ -43,7 +43,8 @@ $ pdflatex test.tex
 
 ## Example with `minted`
 
-First [install Pygments](https://pygments.org/download/). Then save [`lean4.py`](https://raw.githubusercontent.com/leanprover/lean4/master/doc/latex/lean4.py), which contains an version of the Lean highlighter updated for Lean 4, and the following sample LaTeX file `test.tex` into the same directory:
+First [install Pygments](https://pygments.org/download/) (version 2.18 or newer).
+Then save the following sample LaTeX file `test.tex` into the same directory:
 
 ```latex
 \documentclass{article}
@@ -51,9 +52,8 @@ First [install Pygments](https://pygments.org/download/). Then save [`lean4.py`]
 % switch to a monospace font supporting more Unicode characters
 \setmonofont{FreeMono}
 \usepackage{minted}
-% instruct minted to use our local theorem.py
-\newmintinline[lean]{lean4.py:Lean4Lexer -x}{bgcolor=white}
-\newminted[leancode]{lean4.py:Lean4Lexer -x}{fontsize=\footnotesize}
+\newmintinline[lean]{lean4}{bgcolor=white}
+\newminted[leancode]{lean4}{fontsize=\footnotesize}
 \usemintedstyle{tango}  % a nice, colorful theme
 
 \begin{document}
@@ -67,9 +67,6 @@ theorem funext {f₁ f₂ : ∀ (x : α), β x} (h : ∀ x, f₁ x = f₂ x) : f
 \end{document}
 ```
 
-If your version of `minted` is v2.7 or newer, but before v3.0,
-you will additionally need to follow the workaround described in https://github.com/gpoore/minted/issues/360.
-
 You can then compile `test.tex` by executing the following command:
 
 ```bash
@@ -81,11 +78,14 @@ Some remarks:
  - either `xelatex` or `lualatex` is required to handle Unicode characters in the code.
  - `--shell-escape` is needed to allow `xelatex` to execute `pygmentize` in a shell.
  - If the chosen monospace font is missing some Unicode symbols, you can direct them to be displayed using a fallback font or other replacement LaTeX code.
-``` latex
-\usepackage{newunicodechar}
-\newfontfamily{\freeserif}{DejaVu Sans}
-\newunicodechar{✝}{\freeserif{✝}}
-\newunicodechar{𝓞}{\ensuremath{\mathcal{O}}}
-```
- - minted has a "helpful" feature that draws red boxes around characters the chosen lexer doesn't recognize.
- Since the Lean lexer cannot encompass all user-defined syntax, it is advisable to [work around](https://tex.stackexchange.com/a/343506/14563) this feature.
+   ``` latex
+   \usepackage{newunicodechar}
+   \newfontfamily{\freeserif}{DejaVu Sans}
+   \newunicodechar{✝}{\freeserif{✝}}
+   \newunicodechar{𝓞}{\ensuremath{\mathcal{O}}}
+   ```
+ - If you are using an old version of Pygments, you can copy 
+   [`lean.py`](https://raw.githubusercontent.com/pygments/pygments/master/pygments/lexers/lean.py) into your working directory,
+   and use `lean4.py:Lean4Lexer -x` instead of `lean4` above.
+   If your version of `minted` is v2.7 or newer, but before v3.0,
+   you will additionally need to follow the workaround described in https://github.com/gpoore/minted/issues/360.

nix/bootstrap.nix

@@ -170,7 +170,7 @@ rec {
           ln -sf ${lean-all}/* .
         '';
         buildPhase = ''
-          ctest --output-junit test-results.xml --output-on-failure -E 'leancomptest_(doc_example|foreign)' -j$NIX_BUILD_CORES
+          ctest --output-junit test-results.xml --output-on-failure -E 'leancomptest_(doc_example|foreign)|leanlaketest_init' -j$NIX_BUILD_CORES
         '';
         installPhase = ''
           mkdir $out

src/CMakeLists.txt

@@ -11,7 +11,7 @@ project(LEAN CXX C)
 set(LEAN_VERSION_MAJOR 4)
 set(LEAN_VERSION_MINOR 9)
 set(LEAN_VERSION_PATCH 0)
-set(LEAN_VERSION_IS_RELEASE 0)  # This number is 1 in the release revision, and 0 otherwise.
+set(LEAN_VERSION_IS_RELEASE 1)  # This number is 1 in the release revision, and 0 otherwise.
 set(LEAN_SPECIAL_VERSION_DESC "" CACHE STRING "Additional version description like 'nightly-2018-03-11'")
 set(LEAN_VERSION_STRING "${LEAN_VERSION_MAJOR}.${LEAN_VERSION_MINOR}.${LEAN_VERSION_PATCH}")
 if (LEAN_SPECIAL_VERSION_DESC)

src/Init/Data/BitVec/Basic.lean

@@ -534,6 +534,11 @@ def sshiftRight (a : BitVec n) (s : Nat) : BitVec n := .ofInt n (a.toInt >>> s)
 instance {n} : HShiftLeft  (BitVec m) (BitVec n) (BitVec m) := ⟨fun x y => x <<< y.toNat⟩
 instance {n} : HShiftRight (BitVec m) (BitVec n) (BitVec m) := ⟨fun x y => x >>> y.toNat⟩
 
+/-- Auxiliary function for `rotateLeft`, which does not take into account the case where
+the rotation amount is greater than the bitvector width. -/
+def rotateLeftAux (x : BitVec w) (n : Nat) : BitVec w :=
+  x <<< n ||| x >>> (w - n)
+
 /--
 Rotate left for bit vectors. All the bits of `x` are shifted to higher positions, with the top `n`
 bits wrapping around to fill the low bits.
@@ -543,7 +548,15 @@ rotateLeft  0b0011#4 3 = 0b1001
 ```
 SMT-Lib name: `rotate_left` except this operator uses a `Nat` shift amount.
 -/
-def rotateLeft (x : BitVec w) (n : Nat) : BitVec w := x <<< n ||| x >>> (w - n)
+def rotateLeft (x : BitVec w) (n : Nat) : BitVec w := rotateLeftAux x (n % w)
+
+
+/--
+Auxiliary function for `rotateRight`, which does not take into account the case where
+the rotation amount is greater than the bitvector width.
+-/
+def rotateRightAux (x : BitVec w) (n : Nat) : BitVec w :=
+  x >>> n ||| x <<< (w - n)
 
 /--
 Rotate right for bit vectors. All the bits of `x` are shifted to lower positions, with the
@@ -554,7 +567,7 @@ rotateRight 0b01001#5 1 = 0b10100
 ```
 SMT-Lib name: `rotate_right` except this operator uses a `Nat` shift amount.
 -/
-def rotateRight (x : BitVec w) (n : Nat) : BitVec w := x >>> n ||| x <<< (w - n)
+def rotateRight (x : BitVec w) (n : Nat) : BitVec w := rotateRightAux x (n % w)
 
 /--
 Concatenation of bitvectors. This uses the "big endian" convention that the more significant

src/Init/Data/BitVec/Lemmas.lean

@@ -9,6 +9,7 @@ import Init.Data.Bool
 import Init.Data.BitVec.Basic
 import Init.Data.Fin.Lemmas
 import Init.Data.Nat.Lemmas
+import Init.Data.Nat.Mod
 
 namespace BitVec
 
@@ -222,9 +223,21 @@ theorem toInt_eq_toNat_cond (i : BitVec n) :
       if 2*i.toNat < 2^n then
         (i.toNat : Int)
       else
-        (i.toNat : Int) - (2^n : Nat) := by
-  unfold BitVec.toInt
-  split <;> omega
+        (i.toNat : Int) - (2^n : Nat) :=
+  rfl
+
+theorem msb_eq_false_iff_two_mul_lt (x : BitVec w) : x.msb = false ↔ 2 * x.toNat < 2^w := by
+  cases w <;> simp [Nat.pow_succ, Nat.mul_comm _ 2, msb_eq_decide]
+
+theorem msb_eq_true_iff_two_mul_ge (x : BitVec w) : x.msb = true ↔ 2 * x.toNat ≥ 2^w := by
+  simp [← Bool.ne_false_iff, msb_eq_false_iff_two_mul_lt]
+
+/-- Characterize `x.toInt` in terms of `x.msb`. -/
+theorem toInt_eq_msb_cond (x : BitVec w) :
+    x.toInt = if x.msb then (x.toNat : Int) - (2^w : Nat) else (x.toNat : Int) := by
+  simp only [BitVec.toInt, ← msb_eq_false_iff_two_mul_lt]
+  cases x.msb <;> rfl
+
 
 theorem toInt_eq_toNat_bmod (x : BitVec n) : x.toInt = Int.bmod x.toNat (2^n) := by
   simp only [toInt_eq_toNat_cond]
@@ -1043,4 +1056,32 @@ theorem toNat_intMax_eq : (intMax w).toNat = 2^w - 1 := by
     (ofBoolListLE bs).getMsb i = (decide (i < bs.length) && bs.getD (bs.length - 1 - i) false) := by
   simp [getMsb_eq_getLsb]
 
+/-! # Rotate Left -/
+
+/-- rotateLeft is invariant under `mod` by the bitwidth. -/
+@[simp]
+theorem rotateLeft_mod_eq_rotateLeft {x : BitVec w} {r : Nat} :
+    x.rotateLeft (r % w) = x.rotateLeft r := by
+  simp only [rotateLeft, Nat.mod_mod]
+
+/-- `rotateLeft` equals the bit fiddling definition of `rotateLeftAux` when the rotation amount is
+smaller than the bitwidth. -/
+theorem rotateLeft_eq_rotateLeftAux_of_lt {x : BitVec w} {r : Nat} (hr : r < w) :
+    x.rotateLeft r = x.rotateLeftAux r := by
+  simp only [rotateLeft, Nat.mod_eq_of_lt hr]
+
+/-! ## Rotate Right -/
+
+/-- `rotateRight` equals the bit fiddling definition of `rotateRightAux` when the rotation amount is
+smaller than the bitwidth. -/
+theorem rotateRight_eq_rotateRightAux_of_lt {x : BitVec w} {r : Nat} (hr : r < w) :
+    x.rotateRight r = x.rotateRightAux r := by
+  simp only [rotateRight, Nat.mod_eq_of_lt hr]
+
+/-- rotateRight is invariant under `mod` by the bitwidth. -/
+@[simp]
+theorem rotateRight_mod_eq_rotateRight {x : BitVec w} {r : Nat} :
+    x.rotateRight (r % w) = x.rotateRight r := by
+  simp only [rotateRight, Nat.mod_mod]
+
 end BitVec

src/Init/Data/Char.lean

@@ -5,3 +5,4 @@ Authors: Leonardo de Moura
 -/
 prelude
 import Init.Data.Char.Basic
+import Init.Data.Char.Lemmas

src/Init/Data/Char/Lemmas.lean

@@ -0,0 +1,25 @@
+/-
+Copyright (c) 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+Released under Apache 2.0 license as described in the file LICENSE.
+Authors: Leonardo de Moura
+-/
+prelude
+import Init.Data.Char.Basic
+import Init.Data.UInt.Lemmas
+
+namespace Char
+
+theorem le_def {a b : Char} : a ≤ b ↔ a.1 ≤ b.1 := .rfl
+theorem lt_def {a b : Char} : a < b ↔ a.1 < b.1 := .rfl
+theorem lt_iff_val_lt_val {a b : Char} : a < b ↔ a.val < b.val := Iff.rfl
+@[simp] protected theorem not_le {a b : Char} : ¬ a ≤ b ↔ b < a := UInt32.not_le
+@[simp] protected theorem not_lt {a b : Char} : ¬ a < b ↔ b ≤ a := UInt32.not_lt
+@[simp] protected theorem le_refl (a : Char) : a ≤ a := by simp [le_def]
+@[simp] protected theorem lt_irrefl (a : Char) : ¬ a < a := by simp
+protected theorem le_trans {a b c : Char} : a ≤ b → b ≤ c → a ≤ c := UInt32.le_trans
+protected theorem lt_trans {a b c : Char} : a < b → b < c → a < c := UInt32.lt_trans
+protected theorem le_total (a b : Char) : a ≤ b ∨ b ≤ a := UInt32.le_total a.1 b.1
+protected theorem lt_asymm {a b : Char} (h : a < b) : ¬ b < a := UInt32.lt_asymm h
+protected theorem ne_of_lt {a b : Char} (h : a < b) : a ≠ b := Char.ne_of_val_ne (UInt32.ne_of_lt h)
+
+end Char

src/Init/Data/Fin/Lemmas.lean

@@ -1,7 +1,7 @@
 /-
 Copyright (c) 2022 Mario Carneiro. All rights reserved.
 Released under Apache 2.0 license as described in the file LICENSE.
-Authors: Mario Carneiro
+Authors: Mario Carneiro, Leonardo de Moura
 -/
 prelude
 import Init.Data.Fin.Basic
@@ -94,6 +94,18 @@ theorem lt_iff_val_lt_val {a b : Fin n} : a < b ↔ a.val < b.val := Iff.rfl
 
 @[simp] protected theorem not_lt {a b : Fin n} : ¬ a < b ↔ b ≤ a := Nat.not_lt
 
+@[simp] protected theorem le_refl (a : Fin n) : a ≤ a := by simp [le_def]
+
+@[simp] protected theorem lt_irrefl (a : Fin n) : ¬ a < a := by simp
+
+protected theorem le_trans {a b c : Fin n} : a ≤ b → b ≤ c → a ≤ c := Nat.le_trans
+
+protected theorem lt_trans {a b c : Fin n} : a < b → b < c → a < c := Nat.lt_trans
+
+protected theorem le_total (a b : Fin n) : a ≤ b ∨ b ≤ a := Nat.le_total a b
+
+protected theorem lt_asymm {a b : Fin n} (h : a < b) : ¬ b < a := Nat.lt_asymm h
+
 protected theorem ne_of_lt {a b : Fin n} (h : a < b) : a ≠ b := Fin.ne_of_val_ne (Nat.ne_of_lt h)
 
 protected theorem ne_of_gt {a b : Fin n} (h : a < b) : b ≠ a := Fin.ne_of_val_ne (Nat.ne_of_gt h)
@@ -823,27 +835,3 @@ protected theorem zero_mul (k : Fin (n + 1)) : (0 : Fin (n + 1)) * k = 0 := by
   simp [ext_iff, mul_def]
 
 end Fin
-
-namespace USize
-
-@[simp] theorem lt_def {a b : USize} : a < b ↔ a.toNat < b.toNat := .rfl
-
-@[simp] theorem le_def {a b : USize} : a ≤ b ↔ a.toNat ≤ b.toNat := .rfl
-
-@[simp] theorem zero_toNat : (0 : USize).toNat = 0 := Nat.zero_mod _
-
-@[simp] theorem mod_toNat (a b : USize) : (a % b).toNat = a.toNat % b.toNat :=
-  Fin.mod_val ..
-
-@[simp] theorem div_toNat (a b : USize) : (a / b).toNat = a.toNat / b.toNat :=
-  Fin.div_val ..
-
-@[simp] theorem modn_toNat (a : USize) (b : Nat) : (a.modn b).toNat = a.toNat % b :=
-  Fin.modn_val ..
-
-theorem mod_lt (a b : USize) (h : 0 < b) : a % b < b := USize.modn_lt _ (by simp at h; exact h)
-
-theorem toNat.inj : ∀ {a b : USize}, a.toNat = b.toNat → a = b
-  | ⟨_, _⟩, ⟨_, _⟩, rfl => rfl
-
-end USize

src/Init/Data/Int/Order.lean

@@ -96,7 +96,7 @@ protected theorem le_antisymm {a b : Int} (h₁ : a ≤ b) (h₂ : b ≤ a) : a
   have := Int.ofNat.inj <| Int.add_left_cancel <| this.trans (Int.add_zero _).symm
   rw [← hn, Nat.eq_zero_of_add_eq_zero_left this, ofNat_zero, Int.add_zero a]
 
-protected theorem lt_irrefl (a : Int) : ¬a < a := fun H =>
+@[simp] protected theorem lt_irrefl (a : Int) : ¬a < a := fun H =>
   let ⟨n, hn⟩ := lt.dest H
   have : (a+Nat.succ n) = a+0 := by
     rw [hn, Int.add_zero]

src/Init/Data/Nat/Lemmas.lean

@@ -794,6 +794,9 @@ theorem shiftLeft_shiftLeft (m n : Nat) : ∀ k, (m <<< n) <<< k = m <<< (n + k)
   | 0 => rfl
   | k + 1 => by simp [← Nat.add_assoc, shiftLeft_shiftLeft _ _ k, shiftLeft_succ]
 
+@[simp] theorem shiftLeft_shiftRight (x n : Nat) : x <<< n >>> n = x := by
+  rw [Nat.shiftLeft_eq, Nat.shiftRight_eq_div_pow, Nat.mul_div_cancel _ (Nat.two_pow_pos _)]
+
 theorem mul_add_div {m : Nat} (m_pos : m > 0) (x y : Nat) : (m * x + y) / m = x + y / m := by
   match x with
   | 0 => simp

src/Init/Data/Nat/Linear.lean

@@ -714,4 +714,10 @@ theorem Expr.eq_of_toNormPoly_eq (ctx : Context) (e e' : Expr) (h : e.toNormPoly
   simp [Expr.toNormPoly, Poly.norm] at h
   assumption
 
-end Nat.Linear
+end Linear
+
+def elimOffset {α : Sort u} (a b k : Nat) (h₁ : a + k = b + k) (h₂ : a = b → α) : α := by
+  simp_arith at h₁
+  exact h₂ h₁
+
+end Nat

src/Init/Data/String.lean

@@ -6,3 +6,4 @@ Authors: Leonardo de Moura
 prelude
 import Init.Data.String.Basic
 import Init.Data.String.Extra
+import Init.Data.String.Lemmas

src/Init/Data/String/Basic.lean

@@ -24,6 +24,14 @@ instance : LT String :=
 instance decLt (s₁ s₂ : @& String) : Decidable (s₁ < s₂) :=
   List.hasDecidableLt s₁.data s₂.data
 
+@[reducible] protected def le (a b : String) : Prop := ¬ b < a
+
+instance : LE String :=
+  ⟨String.le⟩
+
+instance decLE (s₁ s₂ : String) : Decidable (s₁ ≤ s₂) :=
+  inferInstanceAs (Decidable (Not _))
+
 /--
 Returns the length of a string in Unicode code points.
 
@@ -178,8 +186,9 @@ Returns the next position in a string after position `p`. If `p` is not a valid
 the result is unspecified.
 
 Examples:
-* `"abc".next ⟨1⟩ = String.Pos.mk 2`
-* `"L∃∀N".next ⟨1⟩ = String.Pos.mk 4`, since `'∃'` is a multi-byte UTF-8 character
+Given `def abc := "abc"` and `def lean := "L∃∀N"`,
+* `abc.get (0 |> abc.next) = 'b'`
+* `lean.get (0 |> lean.next |> lean.next) = '∀'`
 
 Cases where the result is unspecified:
 * `"abc".next ⟨3⟩`, since `3 = s.endPos`
@@ -196,16 +205,52 @@ def utf8PrevAux : List Char → Pos → Pos → Pos
     let i' := i + c
     if i' = p then i else utf8PrevAux cs i' p
 
+/--
+Returns the position in a string before a specified position, `p`. If `p = ⟨0⟩`, returns `0`.
+If `p` is not a valid position, the result is unspecified.
+
+Examples:
+Given `def abc := "abc"` and `def lean := "L∃∀N"`,
+* `abc.get (abc.endPos |> abc.prev) = 'c'`
+* `lean.get (lean.endPos |> lean.prev |> lean.prev |> lean.prev) = '∃'`
+* `"L∃∀N".prev ⟨3⟩` is unspecified, since byte 3 occurs in the middle of the multi-byte character `'∃'`.
+-/
 @[extern "lean_string_utf8_prev"]
 def prev : (@& String) → (@& Pos) → Pos
   | ⟨s⟩, p => if p = 0 then 0 else utf8PrevAux s 0 p
 
+/--
+Returns the first character in `s`. If `s = ""`, returns `(default : Char)`.
+
+Examples:
+* `"abc".front = 'a'`
+* `"".front = (default : Char)`
+-/
 def front (s : String) : Char :=
   get s 0
 
+/--
+Returns the last character in `s`. If `s = ""`, returns `(default : Char)`.
+
+Examples:
+* `"abc".back = 'c'`
+* `"".back = (default : Char)`
+-/
 def back (s : String) : Char :=
   get s (prev s s.endPos)
 
+/--
+Returns `true` if a specified position is greater than or equal to the position which
+points to the end of a string. Otherwise, returns `false`.
+
+Examples:
+Given `def abc := "abc"` and `def lean := "L∃∀N"`,
+* `(0 |> abc.next |> abc.next |> abc.atEnd) = false`
+* `(0 |> abc.next |> abc.next |> abc.next |> abc.next |> abc.atEnd) = true`
+* `(0 |> lean.next |> lean.next |> lean.next |> lean.next |> lean.atEnd) = true`
+
+Because `"L∃∀N"` contains multi-byte characters, `lean.next (lean.next 0)` is not equal to `abc.next (abc.next 0)`.
+-/
 @[extern "lean_string_utf8_at_end"]
 def atEnd : (@& String) → (@& Pos) → Bool
   | s, p => p.byteIdx ≥ utf8ByteSize s
@@ -906,6 +951,10 @@ def beq (ss1 ss2 : Substring) : Bool :=
 
 instance hasBeq : BEq Substring := ⟨beq⟩
 
+/-- Checks whether two substrings have the same position and content. -/
+def sameAs (ss1 ss2 : Substring) : Bool :=
+  ss1.startPos == ss2.startPos && ss1 == ss2
+
 end Substring
 
 namespace String

src/Init/Data/String/Extra.lean

@@ -198,4 +198,35 @@ def removeLeadingSpaces (s : String) : String :=
   let n := findLeadingSpacesSize s
   if n == 0 then s else removeNumLeadingSpaces n s
 
+/--
+Replaces each `\r\n` with `\n` to normalize line endings,
+but does not validate that there are no isolated `\r` characters.
+It is an optimized version of `String.replace text "\r\n" "\n"`.
+-/
+def crlfToLf (text : String) : String :=
+  go "" 0 0
+where
+  go (acc : String) (accStop pos : String.Pos) : String :=
+    if h : text.atEnd pos then
+      -- note: if accStop = 0 then acc is empty
+      if accStop = 0 then text else acc ++ text.extract accStop pos
+    else
+      let c := text.get' pos h
+      let pos' := text.next' pos h
+      if h' : ¬ text.atEnd pos' ∧ c == '\r' ∧ text.get pos' == '\n' then
+        let acc := acc ++ text.extract accStop pos
+        go acc pos' (text.next' pos' h'.1)
+      else
+        go acc accStop pos'
+  termination_by text.utf8ByteSize - pos.byteIdx
+  decreasing_by
+    decreasing_with
+      show text.utf8ByteSize - (text.next' (text.next' pos _) _).byteIdx < text.utf8ByteSize - pos.byteIdx
+      have k := Nat.gt_of_not_le <| mt decide_eq_true h
+      exact Nat.sub_lt_sub_left k (Nat.lt_trans (String.lt_next text pos) (String.lt_next _ _))
+    decreasing_with
+      show text.utf8ByteSize - (text.next' pos _).byteIdx < text.utf8ByteSize - pos.byteIdx
+      have k := Nat.gt_of_not_le <| mt decide_eq_true h
+      exact Nat.sub_lt_sub_left k (String.lt_next _ _)
+
 end String

src/Init/Data/String/Lemmas.lean

@@ -0,0 +1,21 @@
+/-
+Copyright (c) 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+Released under Apache 2.0 license as described in the file LICENSE.
+Authors: Leonardo de Moura
+-/
+prelude
+import Init.Data.Char.Lemmas
+
+namespace String
+
+protected theorem data_eq_of_eq {a b : String} (h : a = b) : a.data = b.data :=
+  h ▸ rfl
+protected theorem ne_of_data_ne {a b : String} (h : a.data ≠ b.data) : a ≠ b :=
+  fun h' => absurd (String.data_eq_of_eq h') h
+@[simp] protected theorem lt_irrefl (s : String) : ¬ s < s :=
+  List.lt_irrefl' Char.lt_irrefl s.data
+protected theorem ne_of_lt {a b : String} (h : a < b) : a ≠ b := by
+  have := String.lt_irrefl a
+  intro h; subst h; contradiction
+
+end String

src/Init/Data/UInt.lean

@@ -6,3 +6,4 @@ Authors: Henrik Böving
 prelude
 import Init.Data.UInt.Basic
 import Init.Data.UInt.Log2
+import Init.Data.UInt.Lemmas

src/Init/Data/UInt/Basic.lean

@@ -364,6 +364,3 @@ instance (a b : USize) : Decidable (a < b) := USize.decLt a b
 instance (a b : USize) : Decidable (a ≤ b) := USize.decLe a b
 instance : Max USize := maxOfLe
 instance : Min USize := minOfLe
-
-theorem USize.modn_lt {m : Nat} : ∀ (u : USize), m > 0 → USize.toNat (u % m) < m
-  | ⟨u⟩, h => Fin.modn_lt u h

src/Init/Data/UInt/Lemmas.lean

@@ -0,0 +1,66 @@
+/-
+Copyright (c) 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+Released under Apache 2.0 license as described in the file LICENSE.
+Authors: Leonardo de Moura
+-/
+prelude
+import Init.Data.UInt.Basic
+import Init.Data.Fin.Lemmas
+
+set_option hygiene false in
+macro "declare_uint_theorems" typeName:ident : command =>
+`(
+namespace $typeName
+
+instance : Inhabited $typeName where
+  default := 0
+
+theorem zero_def : (0 : $typeName) = ⟨0⟩ := rfl
+theorem one_def : (1 : $typeName) = ⟨1⟩ := rfl
+theorem sub_def (a b : $typeName) : a - b = ⟨a.val - b.val⟩ := rfl
+theorem mul_def (a b : $typeName) : a * b = ⟨a.val * b.val⟩ := rfl
+theorem mod_def (a b : $typeName) : a % b = ⟨a.val % b.val⟩ := rfl
+theorem add_def (a b : $typeName) : a + b = ⟨a.val + b.val⟩ := rfl
+
+@[simp] theorem mk_val_eq : ∀ (a : $typeName), mk a.val = a
+  | ⟨_, _⟩ => rfl
+theorem val_eq_of_lt {a : Nat} : a < size → ((ofNat a).val : Nat) = a :=
+  Nat.mod_eq_of_lt
+
+theorem le_def {a b : $typeName} : a ≤ b ↔ a.1 ≤ b.1 := .rfl
+theorem lt_def {a b : $typeName} : a < b ↔ a.1 < b.1 := .rfl
+theorem lt_iff_val_lt_val {a b : $typeName} : a < b ↔ a.val < b.val := .rfl
+@[simp] protected theorem not_le {a b : $typeName} : ¬ a ≤ b ↔ b < a := Fin.not_le
+@[simp] protected theorem not_lt {a b : $typeName} : ¬ a < b ↔ b ≤ a := Fin.not_lt
+@[simp] protected theorem le_refl (a : $typeName) : a ≤ a := by simp [le_def]
+@[simp] protected theorem lt_irrefl (a : $typeName) : ¬ a < a := by simp
+protected theorem le_trans {a b c : $typeName} : a ≤ b → b ≤ c → a ≤ c := Fin.le_trans
+protected theorem lt_trans {a b c : $typeName} : a < b → b < c → a < c := Fin.lt_trans
+protected theorem le_total (a b : $typeName) : a ≤ b ∨ b ≤ a := Fin.le_total a.1 b.1
+protected theorem lt_asymm {a b : $typeName} (h : a < b) : ¬ b < a := Fin.lt_asymm h
+protected theorem val_eq_of_eq {a b : $typeName} (h : a = b) : a.val = b.val := h ▸ rfl
+protected theorem eq_of_val_eq {a b : $typeName} (h : a.val = b.val) : a = b := by cases a; cases b; simp at h; simp [h]
+open $typeName (val_eq_of_eq) in
+protected theorem ne_of_val_ne {a b : $typeName} (h : a.val ≠ b.val) : a ≠ b := fun h' => absurd (val_eq_of_eq h') h
+open $typeName (ne_of_val_ne) in
+protected theorem ne_of_lt {a b : $typeName} (h : a < b) : a ≠ b := ne_of_val_ne (Fin.ne_of_lt h)
+
+@[simp] protected theorem zero_toNat : (0 : $typeName).toNat = 0 := Nat.zero_mod _
+@[simp] protected theorem mod_toNat (a b : $typeName) : (a % b).toNat = a.toNat % b.toNat := Fin.mod_val ..
+@[simp] protected theorem div_toNat (a b : $typeName) : (a / b).toNat = a.toNat / b.toNat := Fin.div_val ..
+@[simp] protected theorem modn_toNat (a : $typeName) (b : Nat) : (a.modn b).toNat = a.toNat % b := Fin.modn_val ..
+protected theorem modn_lt {m : Nat} : ∀ (u : $typeName), m > 0 → toNat (u % m) < m
+  | ⟨u⟩, h => Fin.modn_lt u h
+open $typeName (modn_lt) in
+protected theorem mod_lt (a b : $typeName) (h : 0 < b) : a % b < b := modn_lt _ (by simp [lt_def] at h; exact h)
+protected theorem toNat.inj : ∀ {a b : $typeName}, a.toNat = b.toNat → a = b
+  | ⟨_, _⟩, ⟨_, _⟩, rfl => rfl
+
+end $typeName
+)
+
+declare_uint_theorems UInt8
+declare_uint_theorems UInt16
+declare_uint_theorems UInt32
+declare_uint_theorems UInt64
+declare_uint_theorems USize

src/Init/Grind.lean

@@ -6,3 +6,5 @@ Authors: Leonardo de Moura
 prelude
 import Init.Grind.Norm
 import Init.Grind.Tactics
+import Init.Grind.Lemmas
+import Init.Grind.Cases

src/Init/Grind/Cases.lean

@@ -0,0 +1,15 @@
+/-
+Copyright (c) 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+Released under Apache 2.0 license as described in the file LICENSE.
+Authors: Leonardo de Moura
+-/
+prelude
+import Init.Core
+
+attribute [grind_cases] And Prod False Empty True Unit Exists
+
+namespace Lean.Grind.Eager
+
+attribute [scoped grind_cases] Or
+
+end Lean.Grind.Eager

src/Init/Grind/Lemmas.lean

@@ -0,0 +1,14 @@
+/-
+Copyright (c) 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+Released under Apache 2.0 license as described in the file LICENSE.
+Authors: Leonardo de Moura
+-/
+prelude
+import Init.Core
+
+namespace Lean.Grind
+
+theorem intro_with_eq (p p' q : Prop) (he : p = p') (h : p' → q) : p → q :=
+  fun hp => h (he.mp hp)
+
+end Lean.Grind

src/Init/Grind/Tactics.lean

@@ -7,8 +7,19 @@ prelude
 import Init.Tactics
 
 namespace Lean.Grind
+/--
+The configuration for `grind`.
+Passed to `grind` using, for example, the `grind (config := { eager := true })` syntax.
+-/
+structure Config where
+  /--
+  When `eager` is true (default: `false`), `grind` eagerly splits `if-then-else` and `match`
+  expressions.
+  -/
+  eager : Bool := false
+  deriving Inhabited, BEq
+
 /-!
 `grind` tactic and related tactics.
 -/
-
 end Lean.Grind

src/Init/NotationExtra.lean

@@ -87,6 +87,7 @@ macro:35 xs:bracketedExplicitBinders " × " b:term:35  : term => expandBrackedBi
 macro:35 xs:bracketedExplicitBinders " ×' " b:term:35 : term => expandBrackedBinders ``PSigma xs b
 end
 
+namespace Lean
 -- first step of a `calc` block
 syntax calcFirstStep := ppIndent(colGe term (" := " term)?)
 -- enforce indentation of calc steps so we know when to stop parsing them
@@ -136,6 +137,7 @@ syntax (name := calcTactic) "calc" calcSteps : tactic
 @[inherit_doc «calc»]
 macro tk:"calc" steps:calcSteps : conv =>
   `(conv| tactic => calc%$tk $steps)
+end Lean
 
 @[app_unexpander Unit.unit] def unexpandUnit : Lean.PrettyPrinter.Unexpander
   | `($(_)) => `(())
@@ -361,6 +363,7 @@ macro_rules
   | `(letI $_:ident $_* : $_ := $_; $_) => Lean.Macro.throwUnsupported -- handled by elab
 
 
+namespace Lean
 syntax cdotTk := patternIgnore("· " <|> ". ")
 /-- `· tac` focuses on the main goal and tries to solve it using `tac`, or else fails. -/
 syntax (name := cdot) cdotTk tacticSeqIndentGt : tactic
@@ -368,12 +371,11 @@ syntax (name := cdot) cdotTk tacticSeqIndentGt : tactic
 /--
   Similar to `first`, but succeeds only if one the given tactics solves the current goal.
 -/
-syntax (name := solve) "solve" withPosition((ppDedent(ppLine) colGe "| " tacticSeq)+) : tactic
+syntax (name := solveTactic) "solve" withPosition((ppDedent(ppLine) colGe "| " tacticSeq)+) : tactic
 
 macro_rules
   | `(tactic| solve $[| $ts]* ) => `(tactic| focus first $[| ($ts); done]*)
 
-namespace Lean
 /-! # `repeat` and `while` notation -/
 
 inductive Loop where

src/Init/Prelude.lean

@@ -3644,6 +3644,17 @@ def getPos? (info : SourceInfo) (canonicalOnly := false) : Option String.Pos :=
   | synthetic (pos := pos) .., false => some pos
   | _,                         _     => none
 
+/--
+Gets the end position information from a `SourceInfo`, if available.
+If `originalOnly` is true, then `.synthetic` syntax will also return `none`.
+-/
+def getTailPos? (info : SourceInfo) (canonicalOnly := false) : Option String.Pos :=
+  match info, canonicalOnly with
+  | original (endPos := endPos) ..,  _
+  | synthetic (endPos := endPos) (canonical := true) .., _
+  | synthetic (endPos := endPos) .., false => some endPos
+  | _,                               _     => none
+
 end SourceInfo
 
 /--

src/Init/System/IO.lean

@@ -264,6 +264,13 @@ local macro "nonempty_list" : tactic =>
 /-- Helper method for implementing "deterministic" timeouts. It is the number of "small" memory allocations performed by the current execution thread. -/
 @[extern "lean_io_get_num_heartbeats"] opaque getNumHeartbeats : BaseIO Nat
 
+/--
+Adjusts the heartbeat counter of the current thread by the given amount. This can be useful to give
+allocation-avoiding code additional "weight" and is also used to adjust the counter after resuming
+from a snapshot.
+-/
+@[extern "lean_io_add_heartbeats"] opaque addHeartbeats (count : UInt64) : BaseIO Unit
+
 /--
 The mode of a file handle (i.e., a set of `open` flags and an `fdopen` mode).
 
@@ -786,6 +793,32 @@ instance : MonadLift (ST IO.RealWorld) BaseIO := ⟨id⟩
 def mkRef (a : α) : BaseIO (IO.Ref α) :=
   ST.mkRef a
 
+/--
+Mutable cell that can be passed around for purposes of cooperative task cancellation: request
+cancellation with `CancelToken.set` and check for it with `CancelToken.isSet`.
+
+This is a more flexible alternative to `Task.cancel` as the token can be shared between multiple
+tasks.
+-/
+structure CancelToken where
+  private ref : IO.Ref Bool
+
+namespace CancelToken
+
+/-- Creates a new cancellation token. -/
+def new : BaseIO CancelToken :=
+  CancelToken.mk <$> IO.mkRef false
+
+/-- Activates a cancellation token. Idempotent. -/
+def set (tk : CancelToken) : BaseIO Unit :=
+  tk.ref.set true
+
+/-- Checks whether the cancellation token has been activated. -/
+def isSet (tk : CancelToken) : BaseIO Bool :=
+  tk.ref.get
+
+end CancelToken
+
 namespace FS
 namespace Stream
 

src/Lean/CoreM.lean

@@ -11,6 +11,7 @@ import Lean.Eval
 import Lean.ResolveName
 import Lean.Elab.InfoTree.Types
 import Lean.MonadEnv
+import Lean.Elab.Exception
 
 namespace Lean
 register_builtin_option diagnostics : Bool := {
@@ -85,6 +86,13 @@ structure Context where
   Use the `set_option diag true` to set it to true.
   -/
   diag           : Bool := false
+  /-- If set, used to cancel elaboration from outside when results are not needed anymore. -/
+  cancelTk?      : Option IO.CancelToken := none
+  /--
+  If set (when `showPartialSyntaxErrors` is not set and parsing failed), suppresses most elaboration
+  errors; see also `logMessage` below.
+  -/
+  suppressElabErrors : Bool := false
   deriving Nonempty
 
 /-- CoreM is a monad for manipulating the Lean environment.
@@ -201,16 +209,45 @@ instance : MonadTrace CoreM where
   getTraceState := return (← get).traceState
   modifyTraceState f := modify fun s => { s with traceState := f s.traceState }
 
-/-- Restore backtrackable parts of the state. -/
-def restore (b : State) : CoreM Unit :=
-  modify fun s => { s with env := b.env, messages := b.messages, infoState := b.infoState }
+structure SavedState extends State where
+  /-- Number of heartbeats passed inside `withRestoreOrSaveFull`, not used otherwise. -/
+  passedHearbeats : Nat
+deriving Nonempty
+
+def saveState : CoreM SavedState := do
+  let s ← get
+  return { toState := s, passedHearbeats := 0 }
 
 /--
-Restores full state including sources for unique identifiers. Only intended for incremental reuse
-between elaboration runs, not for backtracking within a single run.
+Incremental reuse primitive: if `reusableResult?` is `none`, runs `cont` with an action `save` that
+on execution returns the saved monadic state at this point including the heartbeats used by `cont`
+so far. If `reusableResult?` on the other hand is `some (a, state)`, restores full `state` including
+heartbeats used and returns `a`.
+
+The intention is for steps that support incremental reuse to initially pass `none` as
+`reusableResult?` and call `save` as late as possible in `cont`. In a further run, if reuse is
+possible, `reusableResult?` should be set to the previous state and result, ensuring that the state
+after running `withRestoreOrSaveFull` is identical in both runs. Note however that necessarily this
+is only an approximation in the case of heartbeats as heartbeats used by `withRestoreOrSaveFull`, by
+the remainder of `cont` after calling `save`, as well as by reuse-handling code such as the one
+supplying `reusableResult?` are not accounted for.
 -/
-def restoreFull (b : State) : CoreM Unit :=
-  set b
+@[specialize] def withRestoreOrSaveFull (reusableResult? : Option (α × SavedState))
+    (cont : (save : CoreM SavedState) → CoreM α) : CoreM α := do
+  if let some (val, state) := reusableResult? then
+    set state.toState
+    IO.addHeartbeats state.passedHearbeats.toUInt64
+    return val
+
+  let startHeartbeats ← IO.getNumHeartbeats
+  cont (do
+    let s ← get
+    let stopHeartbeats ← IO.getNumHeartbeats
+    return { toState := s, passedHearbeats := stopHeartbeats - startHeartbeats })
+
+/-- Restore backtrackable parts of the state. -/
+def SavedState.restore (b : SavedState) : CoreM Unit :=
+  modify fun s => { s with env := b.env, messages := b.messages, infoState := b.infoState }
 
 private def mkFreshNameImp (n : Name) : CoreM Name := do
   let fresh ← modifyGet fun s => (s.nextMacroScope, { s with nextMacroScope := s.nextMacroScope + 1 })
@@ -241,10 +278,18 @@ instance [MetaEval α] : MetaEval (CoreM α) where
 protected def withIncRecDepth [Monad m] [MonadControlT CoreM m] (x : m α) : m α :=
   controlAt CoreM fun runInBase => withIncRecDepth (runInBase x)
 
+builtin_initialize interruptExceptionId : InternalExceptionId ← registerInternalExceptionId `interrupt
+
+/--
+Throws an internal interrupt exception if cancellation has been requested. The exception is not
+caught by `try catch` but is intended to be caught by `Command.withLoggingExceptions` at the top
+level of elaboration. In particular, we want to skip producing further incremental snapshots after
+the exception has been thrown.
+ -/
 @[inline] def checkInterrupted : CoreM Unit := do
-  if (← IO.checkCanceled) then
-    -- should never be visible to users!
-    throw <| Exception.error .missing "elaboration interrupted"
+  if let some tk := (← read).cancelTk? then
+    if (← tk.isSet) then
+      throw <| .internal interruptExceptionId
 
 register_builtin_option debug.moduleNameAtTimeout : Bool := {
   defValue := true
@@ -289,11 +334,13 @@ def getMessageLog : CoreM MessageLog :=
   return (← get).messages
 
 /--
-Returns the current log and then resets its messages but does NOT reset `MessageLog.hadErrors`. Used
+Returns the current log and then resets its messages while adjusting `MessageLog.hadErrors`. Used
 for incremental reporting during elaboration of a single command.
 -/
 def getAndEmptyMessageLog : CoreM MessageLog :=
-  modifyGet fun log => ({ log with msgs := {} }, log)
+  modifyGet fun s => (s.messages, { s with
+    messages.unreported := {}
+    messages.hadErrors  := s.messages.hasErrors })
 
 instance : MonadLog CoreM where
   getRef      := getRef
@@ -301,6 +348,12 @@ instance : MonadLog CoreM where
   getFileName := return (← read).fileName
   hasErrors   := return (← get).messages.hasErrors
   logMessage msg := do
+    if (← read).suppressElabErrors then
+      -- discard elaboration errors, except for a few important and unlikely misleading ones, on
+      -- parse error
+      unless msg.data.hasTag (· matches `Elab.synthPlaceholder | `Tactic.unsolvedGoals) do
+        return
+
     let ctx ← read
     let msg := { msg with data := MessageData.withNamingContext { currNamespace := ctx.currNamespace, openDecls := ctx.openDecls } msg.data };
     modify fun s => { s with messages := s.messages.add msg }
@@ -332,7 +385,8 @@ export Core (CoreM mkFreshUserName checkSystem withCurrHeartbeats)
   We used a similar hack at `Exception.isMaxRecDepth` -/
 def Exception.isMaxHeartbeat (ex : Exception) : Bool :=
   match ex with
-  | Exception.error _ (MessageData.ofFormat (Std.Format.text msg)) => "(deterministic) timeout".isPrefixOf msg
+  | Exception.error _ (MessageData.ofFormatWithInfos ⟨Std.Format.text msg, _⟩) =>
+    "(deterministic) timeout".isPrefixOf msg
   | _ => false
 
 /-- Creates the expression `d → b` -/
@@ -407,19 +461,26 @@ def ImportM.runCoreM (x : CoreM α) : ImportM α := do
   let (a, _) ← (withOptions (fun _ => ctx.opts) x).toIO { fileName := "<ImportM>", fileMap := default } { env := ctx.env }
   return a
 
-/-- Return `true` if the exception was generated by one our resource limits. -/
+/-- Return `true` if the exception was generated by one of our resource limits. -/
 def Exception.isRuntime (ex : Exception) : Bool :=
   ex.isMaxHeartbeat || ex.isMaxRecDepth
 
+/-- Returns `true` if the exception is an interrupt generated by `checkInterrupted`. -/
+def Exception.isInterrupt : Exception → Bool
+  | Exception.internal id _ => id == Core.interruptExceptionId
+  | _ => false
+
 /--
-Custom `try-catch` for all monads based on `CoreM`. We don't want to catch "runtime exceptions"
-in these monads, but on `CommandElabM`. See issues #2775 and #2744 as well as `MonadAlwayExcept`.
+Custom `try-catch` for all monads based on `CoreM`. We usually don't want to catch "runtime
+exceptions" these monads, but on `CommandElabM`. See issues #2775 and #2744 as well as
+`MonadAlwaysExcept`. Also, we never want to catch interrupt exceptions inside the elaborator.
 -/
 @[inline] protected def Core.tryCatch (x : CoreM α) (h : Exception → CoreM α) : CoreM α := do
   try
     x
   catch ex =>
-    if ex.isRuntime then
+    if ex.isInterrupt || ex.isRuntime then
+
       throw ex -- We should use `tryCatchRuntimeEx` for catching runtime exceptions
     else
       h ex

src/Lean/Data/HashMap.lean

@@ -249,6 +249,8 @@ def toArray (m : HashMap α β) : Array (α × β) :=
 def numBuckets (m : HashMap α β) : Nat :=
   m.val.buckets.val.size
 
+variable [BEq α] [Hashable α]
+
 /-- Builds a `HashMap` from a list of key-value pairs. Values of duplicated keys are replaced by their respective last occurrences. -/
 def ofList (l : List (α × β)) : HashMap α β :=
   l.foldl (init := HashMap.empty) (fun m p => m.insert p.fst p.snd)
@@ -260,6 +262,7 @@ def ofListWith (l : List (α × β)) (f : β → β → β) : HashMap α β :=
       match m.find? p.fst with
         | none   => m.insert p.fst p.snd
         | some v => m.insert p.fst $ f v p.snd)
+
 end Lean.HashMap
 
 /--

src/Lean/Data/Position.lean

@@ -106,7 +106,7 @@ def ofPosition (text : FileMap) (pos : Position) : String.Pos :=
 
 /--
 Returns the position of the start of (1-based) line `line`.
-This gives the stame result as `map.ofPosition ⟨line, 0⟩`, but is more efficient.
+This gives the same result as `map.ofPosition ⟨line, 0⟩`, but is more efficient.
 -/
 def lineStart (map : FileMap) (line : Nat) : String.Pos :=
   if h : line - 1 < map.positions.size then

src/Lean/Elab/Calc.lean

@@ -112,10 +112,12 @@ def elabCalcSteps (steps : TSyntax ``calcSteps) : TermElabM Expr := do
   return result?.get!.1
 
 /-- Elaborator for the `calc` term mode variant. -/
-@[builtin_term_elab «calc»]
+@[builtin_term_elab Lean.calc]
 def elabCalc : TermElab := fun stx expectedType? => do
   let steps : TSyntax ``calcSteps := ⟨stx[1]⟩
   let result ← elabCalcSteps steps
   synthesizeSyntheticMVarsUsingDefault
   let result ← ensureHasType expectedType? result
   return result
+
+end Lean.Elab.Term

src/Lean/Elab/Command.lean

@@ -47,8 +47,9 @@ structure Context where
   ref            : Syntax := Syntax.missing
   tacticCache?   : Option (IO.Ref Tactic.Cache)
   /--
-  Snapshot for incremental reuse and reporting of command elaboration. Currently unused in Lean
-  itself.
+  Snapshot for incremental reuse and reporting of command elaboration. Currently only used for
+  (mutual) defs and contained tactics, in which case the `DynamicSnapshot` is a
+  `HeadersParsedSnapshot`.
 
   Definitely resolved in `Language.Lean.process.doElab`.
 
@@ -56,6 +57,13 @@ structure Context where
   old elaboration are identical.
   -/
   snap?          : Option (Language.SnapshotBundle Language.DynamicSnapshot)
+  /-- Cancellation token forwarded to `Core.cancelTk?`. -/
+  cancelTk?      : Option IO.CancelToken
+  /--
+  If set (when `showPartialSyntaxErrors` is not set and parsing failed), suppresses most elaboration
+  errors; see also `logMessage` below.
+  -/
+  suppressElabErrors : Bool := false
 
 abbrev CommandElabCoreM (ε) := ReaderT Context $ StateRefT State $ EIO ε
 abbrev CommandElabM := CommandElabCoreM Exception
@@ -73,6 +81,21 @@ Remark: see comment at TermElabM
 @[always_inline]
 instance : Monad CommandElabM := let i := inferInstanceAs (Monad CommandElabM); { pure := i.pure, bind := i.bind }
 
+/-- Like `Core.tryCatch` but do catch runtime exceptions. -/
+@[inline] protected def tryCatch (x : CommandElabM α) (h : Exception → CommandElabM α) :
+    CommandElabM α := do
+  try
+    x
+  catch ex =>
+    if ex.isInterrupt then
+      throw ex
+    else
+      h ex
+
+instance : MonadExceptOf Exception CommandElabM where
+  throw    := throw
+  tryCatch := Command.tryCatch
+
 def mkState (env : Environment) (messages : MessageLog := {}) (opts : Options := {}) : State := {
   env         := env
   messages    := messages
@@ -160,17 +183,18 @@ private def runCore (x : CoreM α) : CommandElabM α := do
   let env := Kernel.resetDiag s.env
   let scope := s.scopes.head!
   let coreCtx : Core.Context := {
-    fileName       := ctx.fileName
-    fileMap        := ctx.fileMap
-    currRecDepth   := ctx.currRecDepth
-    maxRecDepth    := s.maxRecDepth
-    ref            := ctx.ref
-    currNamespace  := scope.currNamespace
-    openDecls      := scope.openDecls
-    initHeartbeats := heartbeats
-    currMacroScope := ctx.currMacroScope
-    options        := scope.opts
-  }
+    fileName           := ctx.fileName
+    fileMap            := ctx.fileMap
+    currRecDepth       := ctx.currRecDepth
+    maxRecDepth        := s.maxRecDepth
+    ref                := ctx.ref
+    currNamespace      := scope.currNamespace
+    openDecls          := scope.openDecls
+    initHeartbeats     := heartbeats
+    currMacroScope     := ctx.currMacroScope
+    options            := scope.opts
+    cancelTk?          := ctx.cancelTk?
+    suppressElabErrors := ctx.suppressElabErrors }
   let x : EIO _ _ := x.run coreCtx {
     env
     ngen := s.ngen
@@ -215,6 +239,11 @@ instance : MonadLog CommandElabM where
   getFileName := return (← read).fileName
   hasErrors   := return (← get).messages.hasErrors
   logMessage msg := do
+    if (← read).suppressElabErrors then
+      -- discard elaboration errors on parse error
+      -- NOTE: unlike `CoreM`'s `logMessage`, we do not currently have any command-level errors that
+      -- we want to allowlist
+      return
     let currNamespace ← getCurrNamespace
     let openDecls ← getOpenDecls
     let msg := { msg with data := MessageData.withNamingContext { currNamespace := currNamespace, openDecls := openDecls } msg.data }
@@ -321,11 +350,19 @@ partial def elabCommand (stx : Syntax) : CommandElabM Unit := do
 
 builtin_initialize registerTraceClass `Elab.input
 
+/-- Option for showing elaboration errors from partial syntax errors. -/
+register_builtin_option showPartialSyntaxErrors : Bool := {
+  defValue := false
+  descr    := "show elaboration errors from partial syntax trees (i.e. after parser recovery)"
+}
+
 /--
 `elabCommand` wrapper that should be used for the initial invocation, not for recursive calls after
 macro expansion etc.
 -/
 def elabCommandTopLevel (stx : Syntax) : CommandElabM Unit := withRef stx do profileitM Exception "elaboration" (← getOptions) do
+  withReader ({ · with suppressElabErrors :=
+    stx.hasMissing && !showPartialSyntaxErrors.get (← getOptions) }) do
   let initMsgs ← modifyGet fun st => (st.messages, { st with messages := {} })
   let initInfoTrees ← getResetInfoTrees
   try
@@ -462,7 +499,12 @@ def runTermElabM (elabFn : Array Expr → TermElabM α) : CommandElabM α := do
           Term.addAutoBoundImplicits' xs someType fun xs _ =>
             Term.withoutAutoBoundImplicit <| elabFn xs
 
-@[inline] def catchExceptions (x : CommandElabM Unit) : CommandElabCoreM Empty Unit := fun ctx ref =>
+/--
+Catches and logs exceptions occurring in `x`. Unlike `try catch` in `CommandElabM`, this function
+catches interrupt exceptions as well and thus is intended for use at the top level of elaboration.
+Interrupt and abort exceptions are caught but not logged.
+-/
+@[inline] def withLoggingExceptions (x : CommandElabM Unit) : CommandElabCoreM Empty Unit := fun ctx ref =>
   EIO.catchExceptions (withLogging x ctx ref) (fun _ => pure ())
 
 private def liftAttrM {α} (x : AttrM α) : CommandElabM α := do
@@ -528,6 +570,7 @@ def liftCommandElabM (cmd : CommandElabM α) : CoreM α := do
       ref := ← getRef
       tacticCache? := none
       snap? := none
+      cancelTk? := (← read).cancelTk?
     } |>.run {
       env := ← getEnv
       maxRecDepth := ← getMaxRecDepth
@@ -537,7 +580,7 @@ def liftCommandElabM (cmd : CommandElabM α) : CoreM α := do
     traceState.traces := coreState.traceState.traces ++ commandState.traceState.traces
     env := commandState.env
   }
-  if let some err := commandState.messages.msgs.toArray.find? (·.severity matches .error) then
+  if let some err := commandState.messages.toArray.find? (·.severity matches .error) then
     throwError err.data
   pure a
 

src/Lean/Elab/DefView.lean

@@ -28,6 +28,81 @@ def DefKind.isExample : DefKind → Bool
   | .example => true
   | _        => false
 
+/-- Header elaboration data of a `DefView`. -/
+structure DefViewElabHeaderData where
+  /--
+    Short name. Recall that all declarations in Lean 4 are potentially recursive. We use `shortDeclName` to refer
+    to them at `valueStx`, and other declarations in the same mutual block. -/
+  shortDeclName : Name
+  /-- Full name for this declaration. This is the name that will be added to the `Environment`. -/
+  declName      : Name
+  /-- Universe level parameter names explicitly provided by the user. -/
+  levelNames    : List Name
+  /-- Syntax objects for the binders occurring before `:`, we use them to populate the `InfoTree` when elaborating `valueStx`. -/
+  binderIds     : Array Syntax
+  /-- Number of parameters before `:`, it also includes auto-implicit parameters automatically added by Lean. -/
+  numParams     : Nat
+  /-- Type including parameters. -/
+  type          : Expr
+deriving Inhabited
+
+section Snapshots
+open Language
+
+/-- Snapshot after processing of a definition body.  -/
+structure BodyProcessedSnapshot extends Language.Snapshot where
+  /-- State after elaboration. -/
+  state : Term.SavedState
+  /-- Elaboration result. -/
+  value : Expr
+deriving Nonempty
+instance : Language.ToSnapshotTree BodyProcessedSnapshot where
+  toSnapshotTree s := ⟨s.toSnapshot, #[]⟩
+
+/-- Snapshot after elaboration of a definition header. -/
+structure HeaderProcessedSnapshot extends Language.Snapshot where
+  /-- Elaboration results. -/
+  view : DefViewElabHeaderData
+  /-- Resulting elaboration state, including any environment additions. -/
+  state : Term.SavedState
+  /-- Syntax of top-level tactic block if any, for checking reuse of `tacSnap?`. -/
+  tacStx? : Option Syntax
+  /-- Incremental execution of main tactic block, if any. -/
+  tacSnap? : Option (SnapshotTask Tactic.TacticParsedSnapshot)
+  /-- Syntax of definition body, for checking reuse of `bodySnap`. -/
+  bodyStx : Syntax
+  /-- Result of body elaboration. -/
+  bodySnap : SnapshotTask (Option BodyProcessedSnapshot)
+deriving Nonempty
+instance : Language.ToSnapshotTree HeaderProcessedSnapshot where
+  toSnapshotTree s := ⟨s.toSnapshot,
+    (match s.tacSnap? with
+      | some tac => #[tac.map (sync := true) toSnapshotTree]
+      | none     => #[]) ++
+    #[s.bodySnap.map (sync := true) toSnapshotTree]⟩
+
+/-- State before elaboration of a mutual definition. -/
+structure DefParsed where
+  /--
+  Input substring uniquely identifying header elaboration result given the same `Environment`.
+  If missing, results should never be reused.
+  -/
+  headerSubstr? : Option Substring
+  /-- Elaboration result, unless fatal exception occurred. -/
+  headerProcessedSnap : SnapshotTask (Option HeaderProcessedSnapshot)
+deriving Nonempty
+
+/-- Snapshot after syntax tree has been split into separate mutual def headers. -/
+structure DefsParsedSnapshot extends Language.Snapshot where
+  /-- Definitions of this mutual block. -/
+  defs : Array DefParsed
+deriving Nonempty, TypeName
+instance : Language.ToSnapshotTree DefsParsedSnapshot where
+  toSnapshotTree s := ⟨s.toSnapshot,
+    s.defs.map (·.headerProcessedSnap.map (sync := true) toSnapshotTree)⟩
+
+end Snapshots
+
 structure DefView where
   kind          : DefKind
   ref           : Syntax
@@ -36,6 +111,13 @@ structure DefView where
   binders       : Syntax
   type?         : Option Syntax
   value         : Syntax
+  /--
+  Snapshot for incremental processing of this definition.
+
+  Invariant: If the bundle's `old?` is set, then elaboration of the header is guaranteed to result
+  in the same elaboration result and state, i.e. reuse is possible.
+  -/
+  headerSnap?   : Option (Language.SnapshotBundle (Option HeaderProcessedSnapshot)) := none
   deriving?     : Option (Array Syntax) := none
   deriving Inhabited
 

src/Lean/Elab/Exception.lean

@@ -5,7 +5,7 @@ Authors: Leonardo de Moura
 -/
 prelude
 import Lean.InternalExceptionId
-import Lean.Meta.Basic
+import Lean.Exception
 
 namespace Lean.Elab
 

src/Lean/Elab/Extra.lean

@@ -241,7 +241,10 @@ private def hasCoe (fromType toType : Expr) : TermElabM Bool := do
 
 private structure AnalyzeResult where
   max?            : Option Expr := none
-  hasUncomparable : Bool := false -- `true` if there are two types `α` and `β` where we don't have coercions in any direction.
+  /-- `true` if there are two types `α` and `β` where we don't have coercions in any direction. -/
+  hasUncomparable : Bool := false
+  /-- `true` if there are any leaf terms with an unknown type (according to `isUnknown`). -/
+  hasUnknown      : Bool := false
 
 private def isUnknown : Expr → Bool
   | .mvar ..        => true
@@ -255,7 +258,7 @@ private def analyze (t : Tree) (expectedType? : Option Expr) : TermElabM Analyze
     match expectedType? with
     | none => pure none
     | some expectedType =>
-      let expectedType ← instantiateMVars expectedType
+      let expectedType := (← instantiateMVars expectedType).cleanupAnnotations
       if isUnknown expectedType then pure none else pure (some expectedType)
   (go t *> get).run' { max? }
 where
@@ -268,8 +271,10 @@ where
        | .binop _ _ _ lhs rhs => go lhs; go rhs
        | .unop _ _ arg => go arg
        | .term _ _ val =>
-         let type ← instantiateMVars (← inferType val)
-         unless isUnknown type do
+         let type := (← instantiateMVars (← inferType val)).cleanupAnnotations
+         if isUnknown type then
+           modify fun s => { s with hasUnknown := true }
+         else
            match (← get).max? with
            | none     => modify fun s => { s with max? := type }
            | some max =>
@@ -430,7 +435,7 @@ mutual
       | .unop ref f arg =>
         return .unop ref f (← go arg none false false)
       | .term ref trees e =>
-        let type ← instantiateMVars (← inferType e)
+        let type := (← instantiateMVars (← inferType e)).cleanupAnnotations
         trace[Elab.binop] "visiting {e} : {type} =?= {maxType}"
         if isUnknown type then
           if let some f := f? then
@@ -448,12 +453,17 @@ mutual
 
   private partial def toExpr (tree : Tree) (expectedType? : Option Expr) : TermElabM Expr := do
     let r ← analyze tree expectedType?
-    trace[Elab.binop] "hasUncomparable: {r.hasUncomparable}, maxType: {r.max?}"
+    trace[Elab.binop] "hasUncomparable: {r.hasUncomparable}, hasUnknown: {r.hasUnknown}, maxType: {r.max?}"
     if r.hasUncomparable || r.max?.isNone then
       let result ← toExprCore tree
       ensureHasType expectedType? result
     else
       let result ← toExprCore (← applyCoe tree r.max?.get! (isPred := false))
+      unless r.hasUnknown do
+        -- Record the resulting maxType calculation.
+        -- We can do this when all the types are known, since in this case `hasUncomparable` is valid.
+        -- If they're not known, recording maxType like this can lead to heterogeneous operations failing to elaborate.
+        discard <| isDefEqGuarded (← inferType result) r.max?.get!
       trace[Elab.binop] "result: {result}"
       ensureHasType expectedType? result
 
@@ -519,7 +529,7 @@ def elabBinRelCore (noProp : Bool) (stx : Syntax) (expectedType? : Option Expr)
     let rhs ← withRef rhsStx <| toTree rhsStx
     let tree := .binop stx .regular f lhs rhs
     let r ← analyze tree none
-    trace[Elab.binrel] "hasUncomparable: {r.hasUncomparable}, maxType: {r.max?}"
+    trace[Elab.binrel] "hasUncomparable: {r.hasUncomparable}, hasUnknown: {r.hasUnknown}, maxType: {r.max?}"
     if r.hasUncomparable || r.max?.isNone then
       -- Use default elaboration strategy + `toBoolIfNecessary`
       let lhs ← toExprCore lhs

src/Lean/Elab/Frontend.lean

@@ -16,6 +16,7 @@ structure State where
   parserState  : Parser.ModuleParserState
   cmdPos       : String.Pos
   commands     : Array Syntax := #[]
+deriving Nonempty
 
 structure Context where
   inputCtx : Parser.InputContext
@@ -34,6 +35,7 @@ def setCommandState (commandState : Command.State) : FrontendM Unit :=
     fileMap      := ctx.inputCtx.fileMap
     tacticCache? := none
     snap?        := none
+    cancelTk?    := none
   }
   match (← liftM <| EIO.toIO' <| (x cmdCtx).run s.commandState) with
   | Except.error e      => throw <| IO.Error.userError s!"unexpected internal error: {← e.toMessageData.toString}"
@@ -44,15 +46,6 @@ def elabCommandAtFrontend (stx : Syntax) : FrontendM Unit := do
     let initMsgs ← modifyGet fun st => (st.messages, { st with messages := {} })
     Command.elabCommandTopLevel stx
     let mut msgs := (← get).messages
-    -- `stx.hasMissing` should imply `initMsgs.hasErrors`, but the latter should be cheaper to check
-    -- in general
-    if !Language.Lean.showPartialSyntaxErrors.get (← getOptions) && initMsgs.hasErrors &&
-        stx.hasMissing then
-      -- discard elaboration errors, except for a few important and unlikely misleading ones, on
-      -- parse error
-      msgs := ⟨msgs.msgs.filter fun msg =>
-        msg.data.hasTag (fun tag => tag == `Elab.synthPlaceholder ||
-          tag == `Tactic.unsolvedGoals || (`_traceMsg).isSuffixOf tag)⟩
     modify ({ · with messages := initMsgs ++ msgs })
 
 def updateCmdPos : FrontendM Unit := do
@@ -92,6 +85,47 @@ def IO.processCommands (inputCtx : Parser.InputContext) (parserState : Parser.Mo
   let (_, s) ← (Frontend.processCommands.run { inputCtx := inputCtx }).run { commandState := commandState, parserState := parserState, cmdPos := parserState.pos }
   pure s
 
+structure IncrementalState extends State where
+  inputCtx    : Parser.InputContext
+  initialSnap : Language.Lean.CommandParsedSnapshot
+deriving Nonempty
+
+open Language in
+/--
+Variant of `IO.processCommands` that uses the new Lean language processor implementation for
+potential incremental reuse. Pass in result of a previous invocation done with the same state
+(but usually different input context) to allow for reuse.
+-/
+-- `IO.processCommands` can be reimplemented on top of this as soon as the additional tasks speed up
+-- things instead of slowing them down
+partial def IO.processCommandsIncrementally (inputCtx : Parser.InputContext)
+    (parserState : Parser.ModuleParserState) (commandState : Command.State)
+    (old? : Option IncrementalState) :
+    BaseIO IncrementalState := do
+  let task ← Language.Lean.processCommands inputCtx parserState commandState
+    (old?.map fun old => (old.inputCtx, old.initialSnap))
+  go task.get task #[]
+where
+  go initialSnap t commands :=
+    let snap := t.get
+    let commands := commands.push snap.data.stx
+    if let some next := snap.nextCmdSnap? then
+      go initialSnap next commands
+    else
+      -- Opting into reuse also enables incremental reporting, so make sure to collect messages from
+      -- all snapshots
+      let messages := toSnapshotTree initialSnap
+        |>.getAll.map (·.diagnostics.msgLog)
+        |>.foldl (· ++ ·) {}
+      let trees := toSnapshotTree initialSnap
+        |>.getAll.map (·.infoTree?) |>.filterMap id |>.toPArray'
+      return {
+        commandState := { snap.data.finishedSnap.get.cmdState with messages, infoState.trees := trees }
+        parserState := snap.data.parserState
+        cmdPos := snap.data.parserState.pos
+        inputCtx, initialSnap, commands
+      }
+
 def process (input : String) (env : Environment) (opts : Options) (fileName : Option String := none) : IO (Environment × MessageLog) := do
   let fileName   := fileName.getD "<input>"
   let inputCtx   := Parser.mkInputContext input fileName
@@ -113,8 +147,7 @@ def runFrontend
     : IO (Environment × Bool) := do
   let startTime := (← IO.monoNanosNow).toFloat / 1000000000
   let inputCtx := Parser.mkInputContext input fileName
-  -- TODO: replace with `#lang` processing
-  if /- Lean #lang? -/ true then
+  if true then
     -- Temporarily keep alive old cmdline driver for the Lean language so that we don't pay the
     -- overhead of passing the environment between snapshots until we actually make good use of it
     -- outside the server

src/Lean/Elab/MutualDef.lean

@@ -20,28 +20,24 @@ import Lean.Elab.DeclarationRange
 namespace Lean.Elab
 open Lean.Parser.Term
 
-/-- `DefView` after elaborating the header. -/
-structure DefViewElabHeader where
-  ref           : Syntax
-  modifiers     : Modifiers
-  /-- Stores whether this is the header of a definition, theorem, ... -/
-  kind          : DefKind
+open Language
+
+/-- `DefView` plus header elaboration data and snapshot. -/
+structure DefViewElabHeader extends DefView, DefViewElabHeaderData where
   /--
-    Short name. Recall that all declarations in Lean 4 are potentially recursive. We use `shortDeclName` to refer
-    to them at `valueStx`, and other declarations in the same mutual block. -/
-  shortDeclName : Name
-  /-- Full name for this declaration. This is the name that will be added to the `Environment`. -/
-  declName      : Name
-  /-- Universe level parameter names explicitly provided by the user. -/
-  levelNames    : List Name
-  /-- Syntax objects for the binders occurring before `:`, we use them to populate the `InfoTree` when elaborating `valueStx`. -/
-  binderIds     : Array Syntax
-  /-- Number of parameters before `:`, it also includes auto-implicit parameters automatically added by Lean. -/
-  numParams     : Nat
-  /-- Type including parameters. -/
-  type          : Expr
-  /-- `Syntax` object the body/value of the definition. -/
-  valueStx      : Syntax
+  Snapshot for incremental processing of top-level tactic block, if any.
+
+  Invariant: if the bundle's `old?` is set, then the state *up to the start* of the tactic block is
+  unchanged, i.e. reuse is possible.
+  -/
+  tacSnap? : Option (Language.SnapshotBundle Tactic.TacticParsedSnapshot)
+  /--
+  Snapshot for incremental processing of definition body.
+
+  Invariant: if the bundle's `old?` is set, then elaboration of the body is guaranteed to result in
+  the same elaboration result and state, i.e. reuse is possible.
+  -/
+  bodySnap? : Option (Language.SnapshotBundle (Option BodyProcessedSnapshot))
   deriving Inhabited
 
 namespace Term
@@ -127,14 +123,65 @@ private def cleanupOfNat (type : Expr) : MetaM Expr := do
     let eNew := mkApp e.appFn! argArgs[1]!
     return .done eNew
 
-/-- Elaborate only the declaration headers. We have to elaborate the headers first because we support mutually recursive declarations in Lean 4. -/
-private def elabHeaders (views : Array DefView) : TermElabM (Array DefViewElabHeader) := do
+/--
+Elaborates only the declaration view headers. We have to elaborate the headers first because we
+support mutually recursive declarations in Lean 4.
+-/
+private def elabHeaders (views : Array DefView)
+    (bodyPromises : Array (IO.Promise (Option BodyProcessedSnapshot)))
+    (tacPromises : Array (IO.Promise Tactic.TacticParsedSnapshot)) :
+    TermElabM (Array DefViewElabHeader) := do
   let expandedDeclIds ← views.mapM fun view => withRef view.ref do
     Term.expandDeclId (← getCurrNamespace) (← getLevelNames) view.declId view.modifiers
   withAutoBoundImplicitForbiddenPred (fun n => expandedDeclIds.any (·.shortName == n)) do
     let mut headers := #[]
-    for view in views, ⟨shortDeclName, declName, levelNames⟩ in expandedDeclIds do
-      let newHeader ← withRef view.ref do
+    -- Can we reuse the result for a body? For starters, all headers (even those below the body)
+    -- must be reusable
+    let mut reuseBody := views.all (·.headerSnap?.any (·.old?.isSome))
+    for view in views, ⟨shortDeclName, declName, levelNames⟩ in expandedDeclIds,
+        tacPromise in tacPromises, bodyPromise in bodyPromises do
+      let mut reusableResult? := none
+      if let some snap := view.headerSnap? then
+        -- by the `DefView.headerSnap?` invariant, safe to reuse results at this point, so let's
+        -- wait for them!
+        if let some old := snap.old?.bind (·.val.get) then
+          let (tacStx?, newTacTask?) ← mkTacTask view.value tacPromise
+          snap.new.resolve <| some { old with
+            tacStx?
+            tacSnap? := newTacTask?
+            bodyStx := view.value
+            bodySnap := mkBodyTask view.value bodyPromise
+          }
+          -- Transition from `DefView.snap?` to `DefViewElabHeader.tacSnap?` invariant: if all
+          -- headers and all previous bodies could be reused, then the state at the *start* of the
+          -- top-level tactic block (if any) is unchanged
+          let reuseTac := reuseBody
+          -- Transition from `DefView.snap?` to `DefViewElabHeader.bodySnap?` invariant: if all
+          -- headers and all previous bodies could be reused and this body syntax is unchanged, then
+          -- we can reuse the result
+          reuseBody := reuseBody &&
+            view.value.structRangeEqWithTraceReuse (← getOptions) old.bodyStx
+          let header := { old.view, view with
+            -- We should only forward the promise if we are actually waiting on the corresponding
+            -- task; otherwise, diagnostics assigned to it will be lost
+            tacSnap? := guard newTacTask?.isSome *> some {
+              old? := do
+                guard reuseTac
+                some ⟨(← old.tacStx?), (← old.tacSnap?)⟩
+              new := tacPromise
+            }
+            bodySnap? := some {
+              -- no syntax guard to store, we already did the necessary checks
+              old? := guard reuseBody *> pure ⟨.missing, old.bodySnap⟩
+              new := bodyPromise
+            }
+          }
+          reusableResult? := some (header, old.state)
+        else
+          reuseBody := false
+
+      let header ← withRestoreOrSaveFull reusableResult? fun save => do
+        withRef view.ref do
         addDeclarationRanges declName view.ref
         applyAttributesAt declName view.modifiers.attrs .beforeElaboration
         withDeclName declName <| withAutoBoundImplicit <| withLevelNames levelNames <|
@@ -164,21 +211,62 @@ private def elabHeaders (views : Array DefView) : TermElabM (Array DefViewElabHe
               let pendingMVarIds ← getMVars type
               discard <| logUnassignedUsingErrorInfos pendingMVarIds <|
                 getPendindMVarErrorMessage views
-            let newHeader := {
-              ref           := view.ref
-              modifiers     := view.modifiers
-              kind          := view.kind
-              shortDeclName := shortDeclName
-              declName, type, levelNames, binderIds
-              numParams     := xs.size
-              valueStx      := view.value : DefViewElabHeader }
+            let newHeader : DefViewElabHeaderData := {
+              declName, shortDeclName, type, levelNames, binderIds
+              numParams := xs.size
+            }
+            let mut newHeader : DefViewElabHeader := { view, newHeader with
+              bodySnap? := none, tacSnap? := none }
+            if let some snap := view.headerSnap? then
+              let (tacStx?, newTacTask?) ← mkTacTask view.value tacPromise
+              snap.new.resolve <| some {
+                diagnostics :=
+                  (← Language.Snapshot.Diagnostics.ofMessageLog (← Core.getAndEmptyMessageLog))
+                view := newHeader.toDefViewElabHeaderData
+                state := (← save)
+                tacStx?
+                tacSnap? := newTacTask?
+                bodyStx := view.value
+                bodySnap := mkBodyTask view.value bodyPromise
+              }
+              newHeader := { newHeader with
+                -- We should only forward the promise if we are actually waiting on the
+                -- corresponding task; otherwise, diagnostics assigned to it will be lost
+                tacSnap? := guard newTacTask?.isSome *> some { old? := none, new := tacPromise }
+                bodySnap? := some { old? := none, new := bodyPromise }
+              }
             check headers newHeader
             return newHeader
-      headers := headers.push newHeader
+      headers := headers.push header
     return headers
+where
+  getBodyTerm? (stx : Syntax) : Option Syntax :=
+    -- TODO: does not work with partial syntax
+    --| `(Parser.Command.declVal| := $body $_suffix:suffix $[$_where]?) => body
+    guard (stx.isOfKind ``Parser.Command.declValSimple) *> some stx[1]
+
+  /-- Creates snapshot task with appropriate range from body syntax and promise. -/
+  mkBodyTask (body : Syntax) (new : IO.Promise (Option BodyProcessedSnapshot)) :
+      Language.SnapshotTask (Option BodyProcessedSnapshot) :=
+    let rangeStx := getBodyTerm? body |>.getD body
+    { range? := rangeStx.getRange?, task := new.result }
+
+  /--
+  If `body` allows for incremental tactic reporting and reuse, creates a snapshot task out of the
+  passed promise with appropriate range, otherwise immediately resolves the promise to a dummy
+  value.
+  -/
+  mkTacTask (body : Syntax) (tacPromise : IO.Promise Tactic.TacticParsedSnapshot) :
+      TermElabM (Option Syntax × Option (Language.SnapshotTask Tactic.TacticParsedSnapshot))
+   := do
+    if let some e := getBodyTerm? body then
+      if let `(by $tacs*) := e then
+        return (e, some { range? := mkNullNode tacs |>.getRange?, task := tacPromise.result })
+    tacPromise.resolve default
+    return (none, none)
 
 /--
-  Create auxiliary local declarations `fs` for the given hearders using their `shortDeclName` and `type`, given hearders, and execute `k fs`.
+  Create auxiliary local declarations `fs` for the given headers using their `shortDeclName` and `type`, given headers, and execute `k fs`.
   The new free variables are tagged as `auxDecl`.
   Remark: `fs.size = headers.size`.
 -/
@@ -250,15 +338,44 @@ private def declValToTerminationHint (declVal : Syntax) : TermElabM WF.Terminati
     return .none
 
 private def elabFunValues (headers : Array DefViewElabHeader) : TermElabM (Array Expr) :=
-  headers.mapM fun header => withDeclName header.declName <| withLevelNames header.levelNames do
-    let valStx ← liftMacroM <| declValToTerm header.valueStx
-    forallBoundedTelescope header.type header.numParams fun xs type => do
-      -- Add new info nodes for new fvars. The server will detect all fvars of a binder by the binder's source location.
-      for i in [0:header.binderIds.size] do
-        -- skip auto-bound prefix in `xs`
-        addLocalVarInfo header.binderIds[i]! xs[header.numParams - header.binderIds.size + i]!
-      let val ← elabTermEnsuringType valStx type
-      mkLambdaFVars xs val
+  headers.mapM fun header => do
+    let mut reusableResult? := none
+    if let some snap := header.bodySnap? then
+      if let some old := snap.old? then
+        -- guaranteed reusable as by the `bodySnap?` invariant, so let's wait on the previous
+        -- elaboration
+        if let some old := old.val.get then
+          snap.new.resolve <| some old
+          -- also make sure to reuse tactic snapshots if present so that body reuse does not lead to
+          -- missed tactic reuse on further changes
+          if let some tacSnap := header.tacSnap? then
+            if let some oldTacSnap := tacSnap.old? then
+              tacSnap.new.resolve oldTacSnap.val.get
+          reusableResult? := some (old.value, old.state)
+
+    withRestoreOrSaveFull reusableResult? fun save => do
+      withDeclName header.declName <| withLevelNames header.levelNames do
+      let valStx ← liftMacroM <| declValToTerm header.value
+      forallBoundedTelescope header.type header.numParams fun xs type => do
+        -- Add new info nodes for new fvars. The server will detect all fvars of a binder by the binder's source location.
+        for i in [0:header.binderIds.size] do
+          -- skip auto-bound prefix in `xs`
+          addLocalVarInfo header.binderIds[i]! xs[header.numParams - header.binderIds.size + i]!
+        let val ← withReader ({ · with tacSnap? := header.tacSnap? }) do
+          -- synthesize mvars here to force the top-level tactic block (if any) to run
+          elabTermEnsuringType valStx type <* synthesizeSyntheticMVarsNoPostponing
+        -- NOTE: without this `instantiatedMVars`, `mkLambdaFVars` may leave around a redex that
+        -- leads to more section variables being included than necessary
+        let val ← instantiateMVars val
+        let val ← mkLambdaFVars xs val
+        if let some snap := header.bodySnap? then
+          snap.new.resolve <| some {
+            diagnostics :=
+              (← Language.Snapshot.Diagnostics.ofMessageLog (← Core.getAndEmptyMessageLog))
+            state := (← save)
+            value := val
+          }
+        return val
 
 private def collectUsed (headers : Array DefViewElabHeader) (values : Array Expr) (toLift : List LetRecToLift)
     : StateRefT CollectFVars.State MetaM Unit := do
@@ -640,7 +757,7 @@ def pushMain (preDefs : Array PreDefinition) (sectionVars : Array Expr) (mainHea
     : TermElabM (Array PreDefinition) :=
   mainHeaders.size.foldM (init := preDefs) fun i preDefs => do
     let header := mainHeaders[i]!
-    let termination ← declValToTerminationHint header.valueStx
+    let termination ← declValToTerminationHint header.value
     let termination := termination.rememberExtraParams header.numParams mainVals[i]!
     let value ← mkLambdaFVars sectionVars mainVals[i]!
     let type ← mkForallFVars sectionVars header.type
@@ -796,38 +913,40 @@ def elabMutualDef (vars : Array Expr) (views : Array DefView) : TermElabM Unit :
   else
     go
 where
-  go := do
-    let scopeLevelNames ← getLevelNames
-    let headers ← elabHeaders views
-    let headers ← levelMVarToParamHeaders views headers
-    let allUserLevelNames := getAllUserLevelNames headers
-    withFunLocalDecls headers fun funFVars => do
-      for view in views, funFVar in funFVars do
-        addLocalVarInfo view.declId funFVar
-      let values ←
-        try
-          let values ← elabFunValues headers
-          Term.synthesizeSyntheticMVarsNoPostponing
-          values.mapM (instantiateMVars ·)
-        catch ex =>
-          logException ex
-          headers.mapM fun header => mkSorry header.type (synthetic := true)
-      let headers ← headers.mapM instantiateMVarsAtHeader
-      let letRecsToLift ← getLetRecsToLift
-      let letRecsToLift ← letRecsToLift.mapM instantiateMVarsAtLetRecToLift
-      checkLetRecsToLiftTypes funFVars letRecsToLift
-      withUsed vars headers values letRecsToLift fun vars => do
-        let preDefs ← MutualClosure.main vars headers funFVars values letRecsToLift
-        for preDef in preDefs do
-          trace[Elab.definition] "{preDef.declName} : {preDef.type} :=\n{preDef.value}"
-        let preDefs ← withLevelNames allUserLevelNames <| levelMVarToParamPreDecls preDefs
-        let preDefs ← instantiateMVarsAtPreDecls preDefs
-        let preDefs ← fixLevelParams preDefs scopeLevelNames allUserLevelNames
-        for preDef in preDefs do
-          trace[Elab.definition] "after eraseAuxDiscr, {preDef.declName} : {preDef.type} :=\n{preDef.value}"
-        checkForHiddenUnivLevels allUserLevelNames preDefs
-        addPreDefinitions preDefs
-        processDeriving headers
+  go :=
+    withAlwaysResolvedPromises views.size fun bodyPromises =>
+    withAlwaysResolvedPromises views.size fun tacPromises => do
+      let scopeLevelNames ← getLevelNames
+      let headers ← elabHeaders views bodyPromises tacPromises
+      let headers ← levelMVarToParamHeaders views headers
+      let allUserLevelNames := getAllUserLevelNames headers
+      withFunLocalDecls headers fun funFVars => do
+        for view in views, funFVar in funFVars do
+          addLocalVarInfo view.declId funFVar
+        let values ←
+          try
+            let values ← elabFunValues headers
+            Term.synthesizeSyntheticMVarsNoPostponing
+            values.mapM (instantiateMVars ·)
+          catch ex =>
+            logException ex
+            headers.mapM fun header => mkSorry header.type (synthetic := true)
+        let headers ← headers.mapM instantiateMVarsAtHeader
+        let letRecsToLift ← getLetRecsToLift
+        let letRecsToLift ← letRecsToLift.mapM instantiateMVarsAtLetRecToLift
+        checkLetRecsToLiftTypes funFVars letRecsToLift
+        withUsed vars headers values letRecsToLift fun vars => do
+          let preDefs ← MutualClosure.main vars headers funFVars values letRecsToLift
+          for preDef in preDefs do
+            trace[Elab.definition] "{preDef.declName} : {preDef.type} :=\n{preDef.value}"
+          let preDefs ← withLevelNames allUserLevelNames <| levelMVarToParamPreDecls preDefs
+          let preDefs ← instantiateMVarsAtPreDecls preDefs
+          let preDefs ← fixLevelParams preDefs scopeLevelNames allUserLevelNames
+          for preDef in preDefs do
+            trace[Elab.definition] "after eraseAuxDiscr, {preDef.declName} : {preDef.type} :=\n{preDef.value}"
+          checkForHiddenUnivLevels allUserLevelNames preDefs
+          addPreDefinitions preDefs
+          processDeriving headers
 
   processDeriving (headers : Array DefViewElabHeader) := do
     for header in headers, view in views do
@@ -842,12 +961,45 @@ end Term
 namespace Command
 
 def elabMutualDef (ds : Array Syntax) : CommandElabM Unit := do
-  let views ← ds.mapM fun d => do
-    let modifiers ← elabModifiers d[0]
-    if ds.size > 1 && modifiers.isNonrec then
-      throwErrorAt d "invalid use of 'nonrec' modifier in 'mutual' block"
-    mkDefView modifiers d[1]
-  runTermElabM fun vars => Term.elabMutualDef vars views
+  withAlwaysResolvedPromises ds.size fun headerPromises => do
+    let substr? := (mkNullNode ds).getSubstring?
+    let snap? := (← read).snap?
+    let mut views := #[]
+    let mut defs := #[]
+    for h : i in [0:ds.size], headerPromise in headerPromises do
+      let d := ds[i]
+      let modifiers ← elabModifiers d[0]
+      if ds.size > 1 && modifiers.isNonrec then
+        throwErrorAt d "invalid use of 'nonrec' modifier in 'mutual' block"
+      let mut view ← mkDefView modifiers d[1]
+      if let some snap := snap? then
+        -- overapproximation: includes previous bodies as well
+        let headerSubstr? := return { (← substr?) with stopPos := (← view.value.getPos?) }
+        view := { view with headerSnap? := some {
+          old? := do
+            -- transitioning from `Context.snap?` to `DefView.snap?` invariant: if the elaboration
+            -- context and state are unchanged, and the substring from the beginning of the first
+            -- header to the beginning of the current body is unchanged, then the elaboration result for
+            -- this header (which includes state from elaboration of previous headers!) should be
+            -- unchanged.
+            let old ← snap.old?
+            -- blocking wait, `HeadersParsedSnapshot` (and hopefully others) should be quick
+            let old ← old.val.get.toTyped? DefsParsedSnapshot
+            let oldParsed ← old.defs[i]?
+            guard <| (← headerSubstr?).sameAs (← oldParsed.headerSubstr?)
+            -- no syntax guard to store, we already did the necessary checks
+            return ⟨.missing, oldParsed.headerProcessedSnap⟩
+          new := headerPromise
+        } }
+        defs := defs.push {
+          headerSubstr?
+          headerProcessedSnap := { range? := d.getRange?, task := headerPromise.result }
+        }
+      views := views.push view
+    if let some snap := snap? then
+      -- no non-fatal diagnostics at this point
+      snap.new.resolve <| .ofTyped { defs, diagnostics := .empty : DefsParsedSnapshot }
+    runTermElabM fun vars => Term.elabMutualDef vars views
 
 end Command
 end Lean.Elab

src/Lean/Elab/SyntheticMVars.lean

@@ -324,7 +324,6 @@ mutual
   If `report := false`, then `runTactic` will not capture exceptions nor will report unsolved goals. Unsolved goals become exceptions.
   -/
   partial def runTactic (mvarId : MVarId) (tacticCode : Syntax) (report := true) : TermElabM Unit := withoutAutoBoundImplicit do
-    let code := tacticCode[1]
     instantiateMVarDeclMVars mvarId
     /-
     TODO: consider using `runPendingTacticsAt` at `mvarId` local context and target type.
@@ -346,7 +345,7 @@ mutual
               -- also put an info node on the `by` keyword specifically -- the token may be `canonical` and thus shown in the info
               -- view even though it is synthetic while a node like `tacticCode` never is (#1990)
               withTacticInfoContext tacticCode[0] do
-                evalTactic code
+                withNarrowedArgTacticReuse (argIdx := 1) (evalTactic ·) tacticCode
             synthesizeSyntheticMVars (postpone := .no)
           unless remainingGoals.isEmpty do
             if report then

src/Lean/Elab/Tactic/Basic.lean

@@ -34,10 +34,6 @@ structure Context where
   -/
   recover    : Bool := true
 
-structure SavedState where
-  term   : Term.SavedState
-  tactic : State
-
 abbrev TacticM := ReaderT Context $ StateRefT State TermElabM
 abbrev Tactic  := Syntax → TacticM Unit
 
@@ -100,6 +96,16 @@ def SavedState.restore (b : SavedState) (restoreInfo := false) : TacticM Unit :=
   b.term.restore restoreInfo
   set b.tactic
 
+@[specialize, inherit_doc Core.withRestoreOrSaveFull]
+def withRestoreOrSaveFull (reusableResult? : Option (α × SavedState))
+    (cont : TacticM SavedState → TacticM α) : TacticM α := do
+  if let some (_, state) := reusableResult? then
+    set state.tactic
+  let reusableResult? := reusableResult?.map (fun (val, state) => (val, state.term))
+  controlAt TermElabM fun runInBase =>
+    Term.withRestoreOrSaveFull reusableResult? fun restore =>
+      runInBase <| cont (return { term := (← restore), tactic := (← get) })
+
 protected def getCurrMacroScope : TacticM MacroScope := do pure (← readThe Core.Context).currMacroScope
 protected def getMainModule     : TacticM Name       := do pure (← getEnv).mainModule
 
@@ -432,6 +438,12 @@ def getNameOfIdent' (id : Syntax) : Name :=
 def withCaseRef [Monad m] [MonadRef m] (arrow body : Syntax) (x : m α) : m α :=
   withRef (mkNullNode #[arrow, body]) x
 
+-- TODO: attribute(s)
+builtin_initialize builtinIncrementalTactics : IO.Ref NameSet ← IO.mkRef {}
+
+def registerBuiltinIncrementalTactic (kind : Name) : IO Unit := do
+  builtinIncrementalTactics.modify fun s => s.insert kind
+
 builtin_initialize registerTraceClass `Elab.tactic
 builtin_initialize registerTraceClass `Elab.tactic.backtrack
 

src/Lean/Elab/Tactic/BuiltinTactic.lean

@@ -29,13 +29,95 @@ open Parser.Tactic
 @[builtin_tactic Lean.Parser.Tactic.«done»] def evalDone : Tactic := fun _ =>
   done
 
-@[builtin_tactic seq1] def evalSeq1 : Tactic := fun stx => do
-  let args := stx[0].getArgs
-  for i in [:args.size] do
-    if i % 2 == 0 then
-      evalTactic args[i]!
-    else
-      saveTacticInfoForToken args[i]! -- add `TacticInfo` node for `;`
+open Language in
+/--
+Evaluates a tactic script in form of a syntax node with alternating tactics and separators as
+children.
+ -/
+partial def evalSepTactics : Tactic := goEven
+where
+  -- `stx[0]` is the next tactic step, if any
+  goEven stx := do
+    if stx.getNumArgs == 0 then
+      return
+    let tac := stx[0]
+    /-
+    Each `goEven` step creates three promises under incrementality and reuses their older versions
+    where possible:
+    * `finished` is resolved when `tac` finishes execution; if `tac` is wholly unchanged from the
+      previous version, its state is reused and `tac` execution is skipped. Note that this promise
+      is never turned into a `SnapshotTask` and added to the snapshot tree as incremental reporting
+      is already covered by the next two promises.
+    * `inner` is passed to `tac` if it is marked as supporting incrementality and can be used for
+      reporting and partial reuse inside of it; if the tactic is unsupported or `finished` is wholly
+      reused, it is ignored.
+    * `next` is used as the context when invoking `goOdd` and thus eventually used for the next
+      `goEven` step. Thus, the incremental state of a tactic script is ultimately represented as a
+      chain of `next` snapshots. Its reuse is disabled if `tac` or its following separator are
+      changed in any way.
+    -/
+    let mut oldInner? := none
+    if let some snap := (← readThe Term.Context).tacSnap? then
+      if let some old := snap.old? then
+        let oldParsed := old.val.get
+        oldInner? := oldParsed.next.get? 0 |>.map (⟨oldParsed.data.stx, ·⟩)
+    -- compare `stx[0]` for `finished`/`next` reuse, focus on remainder of script
+    Term.withNarrowedTacticReuse (stx := stx) (fun stx => (stx[0], mkNullNode stx.getArgs[1:])) fun stxs => do
+      let some snap := (← readThe Term.Context).tacSnap?
+        | do evalTactic tac; goOdd stxs
+      let mut reusableResult? := none
+      let mut oldNext? := none
+      if let some old := snap.old? then
+        -- `tac` must be unchanged given the narrow above; let's reuse `finished`'s state!
+        let oldParsed := old.val.get
+        if let some state := oldParsed.data.finished.get.state? then
+          reusableResult? := some (state, state)
+          -- only allow `next` reuse in this case
+          oldNext? := oldParsed.next.get? 1 |>.map (⟨old.stx, ·⟩)
+
+      withAlwaysResolvedPromise fun next => do
+        withAlwaysResolvedPromise fun finished => do
+          withAlwaysResolvedPromise fun inner => do
+            snap.new.resolve <| .mk {
+              stx := tac
+              diagnostics := (← Language.Snapshot.Diagnostics.ofMessageLog
+                (← Core.getAndEmptyMessageLog))
+              finished := finished.result
+            } #[
+              {
+                range? := tac.getRange?
+                task := inner.result },
+              {
+                range? := stxs |>.getRange?
+                task := next.result }]
+            let state ← withRestoreOrSaveFull reusableResult? fun save => do
+              -- allow nested reuse for allowlisted tactics
+              withTheReader Term.Context ({ · with
+                  tacSnap? :=
+                    guard ((← builtinIncrementalTactics.get).contains tac.getKind) *>
+                    some {
+                      old? := oldInner?
+                      new := inner
+                    } }) do
+                evalTactic tac
+              save
+            finished.resolve { state? := state }
+
+        withTheReader Term.Context ({ · with tacSnap? := some {
+          new := next
+          old? := oldNext?
+        } }) do
+          goOdd stxs
+  -- `stx[0]` is the next separator, if any
+  goOdd stx := do
+    if stx.getNumArgs == 0 then
+      return
+    saveTacticInfoForToken stx[0] -- add `TacticInfo` node for `;`
+    -- disable further reuse on separator change as to not reuse wrong `TacticInfo`
+    Term.withNarrowedTacticReuse (fun stx => (stx[0], mkNullNode stx.getArgs[1:])) goEven stx
+
+@[builtin_tactic seq1] def evalSeq1 : Tactic := fun stx =>
+  evalSepTactics stx[0]
 
 @[builtin_tactic paren] def evalParen : Tactic := fun stx =>
   evalTactic stx[1]
@@ -104,26 +186,20 @@ def addCheckpoints (stx : Syntax) : TacticM Syntax := do
   output := output ++ currentCheckpointBlock
   return stx.setArgs output
 
-/-- Evaluate `sepByIndent tactic "; " -/
-def evalSepByIndentTactic (stx : Syntax) : TacticM Unit := do
-  let stx ← addCheckpoints stx
-  for arg in stx.getArgs, i in [:stx.getArgs.size] do
-    if i % 2 == 0 then
-      evalTactic arg
-    else
-      saveTacticInfoForToken arg
-
-@[builtin_tactic tacticSeq1Indented] def evalTacticSeq1Indented : Tactic := fun stx =>
-  evalSepByIndentTactic stx[0]
+builtin_initialize registerBuiltinIncrementalTactic ``tacticSeq1Indented
+@[builtin_tactic tacticSeq1Indented] def evalTacticSeq1Indented : Tactic :=
+  Term.withNarrowedArgTacticReuse (argIdx := 0) evalSepTactics
 
+builtin_initialize registerBuiltinIncrementalTactic ``tacticSeqBracketed
 @[builtin_tactic tacticSeqBracketed] def evalTacticSeqBracketed : Tactic := fun stx => do
   let initInfo ← mkInitialTacticInfo stx[0]
   withRef stx[2] <| closeUsingOrAdmit do
     -- save state before/after entering focus on `{`
     withInfoContext (pure ()) initInfo
-    evalSepByIndentTactic stx[1]
+    Term.withNarrowedArgTacticReuse (argIdx := 1) evalSepTactics stx
 
-@[builtin_tactic cdot] def evalTacticCDot : Tactic := fun stx => do
+builtin_initialize registerBuiltinIncrementalTactic ``cdot
+@[builtin_tactic Lean.cdot] def evalTacticCDot : Tactic := fun stx => do
   -- adjusted copy of `evalTacticSeqBracketed`; we used to use the macro
   -- ``| `(tactic| $cdot:cdotTk $tacs) => `(tactic| {%$cdot ($tacs) }%$cdot)``
   -- but the token antiquotation does not copy trailing whitespace, leading to
@@ -132,7 +208,7 @@ def evalSepByIndentTactic (stx : Syntax) : TacticM Unit := do
   withRef stx[0] <| closeUsingOrAdmit do
     -- save state before/after entering focus on `·`
     withInfoContext (pure ()) initInfo
-    evalSepByIndentTactic stx[1]
+    Term.withNarrowedArgTacticReuse (argIdx := 1) evalTactic stx
 
 @[builtin_tactic Parser.Tactic.focus] def evalFocus : Tactic := fun stx => do
   let mkInfo ← mkInitialTacticInfo stx[0]
@@ -205,8 +281,9 @@ private def getOptRotation (stx : Syntax) : Nat :=
     throwError "failed on all goals"
   setGoals mvarIdsNew.toList
 
-@[builtin_tactic tacticSeq] def evalTacticSeq : Tactic := fun stx =>
-  evalTactic stx[0]
+builtin_initialize registerBuiltinIncrementalTactic ``tacticSeq
+@[builtin_tactic tacticSeq] def evalTacticSeq : Tactic :=
+  Term.withNarrowedArgTacticReuse (argIdx := 0) evalTactic
 
 partial def evalChoiceAux (tactics : Array Syntax) (i : Nat) : TacticM Unit :=
   if h : i < tactics.size then
@@ -392,7 +469,7 @@ def renameInaccessibles (mvarId : MVarId) (hs : TSyntaxArray ``binderIdent) : Ta
 private def getCaseGoals (tag : TSyntax ``binderIdent) : TacticM (MVarId × List MVarId) := do
   let gs ← getUnsolvedGoals
   let g ← if let `(binderIdent| $tag:ident) := tag then
-    let tag := tag.getId
+    let tag := tag.getId.eraseMacroScopes
     let some g ← findTag? gs tag | notFound gs tag
     pure g
   else
@@ -426,16 +503,16 @@ where
     .group <| .nest 2 <|
     .ofFormat .line ++ .joinSep items sep
 
-
+builtin_initialize registerBuiltinIncrementalTactic ``case
 @[builtin_tactic «case»] def evalCase : Tactic
-  | stx@`(tactic| case $[$tag $hs*]|* =>%$arr $tac:tacticSeq) =>
+  | stx@`(tactic| case $[$tag $hs*]|* =>%$arr $tac:tacticSeq1Indented) =>
     for tag in tag, hs in hs do
       let (g, gs) ← getCaseGoals tag
       let g ← renameInaccessibles g hs
       setGoals [g]
       g.setTag Name.anonymous
-      withCaseRef arr tac do
-        closeUsingOrAdmit (withTacticInfoContext stx (evalTactic tac))
+      withCaseRef arr tac <| closeUsingOrAdmit <| withTacticInfoContext stx <|
+        Term.withNarrowedArgTacticReuse (argIdx := 3) (evalTactic ·) stx
       setGoals gs
   | _ => throwUnsupportedSyntax
 

src/Lean/Elab/Tactic/Calc.lean

@@ -11,7 +11,7 @@ namespace Lean.Elab.Tactic
 open Meta
 
 /-- Elaborator for the `calc` tactic mode variant. -/
-@[builtin_tactic calcTactic]
+@[builtin_tactic Lean.calcTactic]
 def evalCalc : Tactic := fun stx => withMainContext do
   let steps : TSyntax ``calcSteps := ⟨stx[1]⟩
   let (val, mvarIds) ← withCollectingNewGoalsFrom (tagSuffix := `calc) do
@@ -32,3 +32,5 @@ def evalCalc : Tactic := fun stx => withMainContext do
     return val
   (← getMainGoal).assign val
   replaceMainGoal mvarIds
+
+end Lean.Elab.Tactic

src/Lean/Elab/Tactic/Induction.lean

@@ -54,23 +54,25 @@ private def getAltDArrow (alt : Syntax) : Syntax :=
 def isHoleRHS (rhs : Syntax) : Bool :=
   rhs.isOfKind ``Parser.Term.syntheticHole || rhs.isOfKind ``Parser.Term.hole
 
-def evalAlt (mvarId : MVarId) (alt : Syntax) (addInfo : TermElabM Unit) (remainingGoals : Array MVarId) : TacticM (Array MVarId) :=
+def evalAlt (mvarId : MVarId) (alt : Syntax) (addInfo : TermElabM Unit) : TacticM Unit :=
   let rhs := getAltRHS alt
   withCaseRef (getAltDArrow alt) rhs do
     if isHoleRHS rhs then
       addInfo
-      let gs' ← mvarId.withContext <| withTacticInfoContext rhs do
+      mvarId.withContext <| withTacticInfoContext rhs do
         let mvarDecl ← mvarId.getDecl
         let val ← elabTermEnsuringType rhs mvarDecl.type
         mvarId.assign val
         let gs' ← getMVarsNoDelayed val
         tagUntaggedGoals mvarDecl.userName `induction gs'.toList
-        pure gs'
-      return remainingGoals ++ gs'
+        setGoals <| (← getGoals) ++ gs'.toList
     else
-      setGoals [mvarId]
-      closeUsingOrAdmit (withTacticInfoContext alt (addInfo *> evalTactic rhs))
-      return remainingGoals
+      let goals ← getGoals
+      try
+        setGoals [mvarId]
+        closeUsingOrAdmit (withTacticInfoContext alt (addInfo *> evalTactic rhs))
+      finally
+        setGoals goals
 
 /-!
   Helper method for creating an user-defined eliminator/recursor application.
@@ -199,6 +201,9 @@ private def getAltNumFields (elimInfo : ElimInfo) (altName : Name) : TermElabM N
       return altInfo.numFields
   throwError "unknown alternative name '{altName}'"
 
+private def isWildcard (altStx : Syntax) : Bool :=
+  getAltName altStx == `_
+
 private def checkAltNames (alts : Array Alt) (altsSyntax : Array Syntax) : TacticM Unit :=
   for i in [:altsSyntax.size] do
     let altStx := altsSyntax[i]!
@@ -229,151 +234,184 @@ private def saveAltVarsInfo (altMVarId : MVarId) (altStx : Syntax) (fvarIds : Ar
           Term.addLocalVarInfo altVars[i]! (mkFVar fvarId)
           i := i + 1
 
-/--
-  If `altsSyntax` is not empty we reorder `alts` using the order the alternatives have been provided
-  in `altsSyntax`. Motivations:
-
-  1- It improves the effectiveness of the `checkpoint` and `save` tactics. Consider the following example:
-  ```lean
-  example (h₁ : p ∨ q) (h₂ : p → x = 0) (h₃ : q → y = 0) : x * y = 0 := by
-    cases h₁ with
-    | inr h =>
-      sleep 5000 -- sleeps for 5 seconds
-      save
-      have : y = 0 := h₃ h
-      -- We can confortably work here
-    | inl h => stop ...
-  ```
-  If we do reorder, the `inl` alternative will be executed first. Moreover, as we type in the `inr` alternative,
-  type errors will "swallow" the `inl` alternative and affect the tactic state at `save` making it ineffective.
-
-  2- The errors are produced in the same order the appear in the code above. This is not super important when using IDEs.
--/
-def reorderAlts (alts : Array Alt) (altsSyntax : Array Syntax) : Array Alt := Id.run do
-  if altsSyntax.isEmpty then
-    return alts
-  else
-    let mut alts := alts
-    let mut result := #[]
-    for altStx in altsSyntax do
-      let altName := getAltName altStx
-      let some i := alts.findIdx? (·.1 == altName) | return result ++ alts
-      result := result.push alts[i]!
-      alts := alts.eraseIdx i
-    return result ++ alts
-
-def evalAlts (elimInfo : ElimInfo) (alts : Array Alt) (optPreTac : Syntax) (altsSyntax : Array Syntax)
+open Language in
+def evalAlts (elimInfo : ElimInfo) (alts : Array Alt) (optPreTac : Syntax) (altStxs : Array Syntax)
     (initialInfo : Info)
     (numEqs : Nat := 0) (numGeneralized : Nat := 0) (toClear : Array FVarId := #[])
     (toTag : Array (Ident × FVarId) := #[]) : TacticM Unit := do
-  let hasAlts := altsSyntax.size > 0
+  let hasAlts := altStxs.size > 0
   if hasAlts then
     -- default to initial state outside of alts
     -- HACK: because this node has the same span as the original tactic,
     -- we need to take all the info trees we have produced so far and re-nest them
     -- inside this node as well
     let treesSaved ← getResetInfoTrees
-    withInfoContext ((modifyInfoState fun s => { s with trees := treesSaved }) *> go) (pure initialInfo)
-  else go
+    withInfoContext ((modifyInfoState fun s => { s with trees := treesSaved }) *> goWithInfo) (pure initialInfo)
+  else goWithInfo
 where
-  go := do
-    checkAltNames alts altsSyntax
-    let alts := reorderAlts alts altsSyntax
-    let hasAlts := altsSyntax.size > 0
-    let mut usedWildcard := false
-    let mut subgoals := #[] -- when alternatives are not provided, we accumulate subgoals here
-    let mut altsSyntax := altsSyntax
+  -- continuation in the correct info context
+  goWithInfo := do
+    let hasAlts := altStxs.size > 0
+
+    if hasAlts then
+      if let some tacSnap := (← readThe Term.Context).tacSnap? then
+        -- incrementality: create a new promise for each alternative, resolve current snapshot to
+        -- them, eventually put each of them back in `Context.tacSnap?` in `applyAltStx`
+        withAlwaysResolvedPromise fun finished => do
+          withAlwaysResolvedPromises altStxs.size fun altPromises => do
+            tacSnap.new.resolve <| .mk {
+              -- save all relevant syntax here for comparison with next document version
+              stx := mkNullNode altStxs
+              diagnostics := .empty
+              finished := finished.result
+            } (altStxs.zipWith altPromises fun stx prom =>
+                { range? := stx.getRange?, task := prom.result })
+            goWithIncremental <| altPromises.mapIdx fun i prom => {
+              old? := do
+                let old ← tacSnap.old?
+                -- waiting is fine here: this is the old version of the snapshot resolved above
+                -- immediately at the beginning of the tactic
+                let old := old.val.get
+                -- use old version of `mkNullNode altsSyntax` as guard, will be compared with new
+                -- version and picked apart in `applyAltStx`
+                return ⟨old.data.stx, (← old.next[i]?)⟩
+              new := prom
+            }
+            finished.resolve { state? := (← saveState) }
+        return
+
+    goWithIncremental #[]
+
+  -- continuation in the correct incrementality context
+  goWithIncremental (tacSnaps : Array (SnapshotBundle TacticParsedSnapshot)) := do
+    let hasAlts := altStxs.size > 0
+    let mut alts := alts
+
+    -- initial sanity checks: named cases should be known, wildcards should be last
+    checkAltNames alts altStxs
+
+    /-
+    First process `altsSyntax` in order, removing covered alternatives from `alts`. Previously we
+    did one loop through `alts`, looking up suitable alternatives from `altsSyntax`.
+    Motivations for the change:
+
+    1- It improves the effectiveness of incremental reuse. Consider the following example:
+    ```lean
+    example (h₁ : p ∨ q) (h₂ : p → x = 0) (h₃ : q → y = 0) : x * y = 0 := by
+      cases h₁ with
+      | inr h =>
+        sleep 5000 -- sleeps for 5 seconds
+        save
+        have : y = 0 := h₃ h
+        -- We can comfortably work here
+      | inl h => stop ...
+    ```
+    If we iterated through `alts` instead of `altsSyntax`, the `inl` alternative would be executed
+    first, making partial reuse in `inr` impossible (without support for reuse with position
+    adjustments).
+
+    2- The errors are produced in the same order the appear in the code above. This is not super
+    important when using IDEs.
+    -/
+    for altStxIdx in [0:altStxs.size] do
+      let altStx := altStxs[altStxIdx]!
+      let altName := getAltName altStx
+      if let some i := alts.findIdx? (·.1 == altName) then
+        -- cover named alternative
+        applyAltStx tacSnaps altStxIdx altStx alts[i]!
+        alts := alts.eraseIdx i
+      else if !alts.isEmpty && isWildcard altStx then
+        -- cover all alternatives
+        for alt in alts do
+          applyAltStx tacSnaps altStxIdx altStx alt
+        alts := #[]
+      else
+        throwErrorAt altStx "unused alternative '{altName}'"
+
+    -- now process remaining alternatives; these might either be unreachable or we're in `induction`
+    -- without `with`. In all other cases, remaining alternatives are flagged as errors.
     for { name := altName, info, mvarId := altMVarId } in alts do
       let numFields ← getAltNumFields elimInfo altName
-      let mut isWildcard := false
-      let altStx? ←
-        match altsSyntax.findIdx? (fun alt => getAltName alt == altName) with
-        | some idx =>
-          let altStx := altsSyntax[idx]!
-          altsSyntax := altsSyntax.eraseIdx idx
-          pure (some altStx)
-        | none => match altsSyntax.findIdx? (fun alt => getAltName alt == `_) with
-          | some idx =>
-            isWildcard := true
-            pure (some altsSyntax[idx]!)
-          | none =>
-            pure none
-      match altStx? with
-      | none =>
-        let mut (_, altMVarId) ← altMVarId.introN numFields
-        match (← Cases.unifyEqs? numEqs altMVarId {}) with
-        | none   => pure () -- alternative is not reachable
-        | some (altMVarId', subst) =>
-          altMVarId ← if info.provesMotive then
-            (_, altMVarId) ← altMVarId'.introNP numGeneralized
-            pure altMVarId
-          else
-            pure altMVarId'
-          for fvarId in toClear do
-            altMVarId ← altMVarId.tryClear fvarId
-          altMVarId.withContext do
-            for (stx, fvar) in toTag do
-              Term.addLocalVarInfo stx (subst.get fvar)
-          let altMVarIds ← applyPreTac altMVarId
-          if !hasAlts then
-            -- User did not provide alternatives using `|`
-            subgoals := subgoals ++ altMVarIds.toArray
-          else if altMVarIds.isEmpty then
-            pure ()
-          else
-            logError m!"alternative '{altName}' has not been provided"
-            altMVarIds.forM fun mvarId => admitGoal mvarId
-      | some altStx =>
-        (subgoals, usedWildcard) ← withRef altStx do
-          let altVars := getAltVars altStx
-          let numFieldsToName ← if altHasExplicitModifier altStx then pure numFields else getNumExplicitFields altMVarId numFields
-          if altVars.size > numFieldsToName then
-            logError m!"too many variable names provided at alternative '{altName}', #{altVars.size} provided, but #{numFieldsToName} expected"
-          let mut (fvarIds, altMVarId) ← altMVarId.introN numFields (altVars.toList.map getNameOfIdent') (useNamesForExplicitOnly := !altHasExplicitModifier altStx)
-          -- Delay adding the infos for the pattern LHS because we want them to nest
-          -- inside tacticInfo for the current alternative (in `evalAlt`)
-          let addInfo : TermElabM Unit := do
-            if (← getInfoState).enabled then
-              if let some declName := info.declName? then
-                addConstInfo (getAltNameStx altStx) declName
-              saveAltVarsInfo altMVarId altStx fvarIds
-          let unusedAlt := do
-            addInfo
-            if isWildcard then
-              pure (#[], usedWildcard)
-            else
-              throwError "alternative '{altName}' is not needed"
-          match (← Cases.unifyEqs? numEqs altMVarId {}) with
-          | none => unusedAlt
-          | some (altMVarId', subst) =>
-            altMVarId ← if info.provesMotive then
-              (_, altMVarId) ← altMVarId'.introNP numGeneralized
-              pure altMVarId
-            else
-              pure altMVarId'
-            for fvarId in toClear do
-              altMVarId ← altMVarId.tryClear fvarId
-            altMVarId.withContext do
-              for (stx, fvar) in toTag do
-                Term.addLocalVarInfo stx (subst.get fvar)
-            let altMVarIds ← applyPreTac altMVarId
-            if altMVarIds.isEmpty then
-              unusedAlt
-            else
-              let mut subgoals := subgoals
-              for altMVarId' in altMVarIds do
-                subgoals ← evalAlt altMVarId' altStx addInfo subgoals
-              pure (subgoals, usedWildcard || isWildcard)
-    if usedWildcard then
-      altsSyntax := altsSyntax.filter fun alt => getAltName alt != `_
-    unless altsSyntax.isEmpty do
-      logErrorAt altsSyntax[0]! "unused alternative"
-    setGoals subgoals.toList
+      let mut (_, altMVarId) ← altMVarId.introN numFields
+      let some (altMVarId', subst) ← Cases.unifyEqs? numEqs altMVarId {}
+        | continue  -- alternative is not reachable
+      altMVarId ← if info.provesMotive then
+        (_, altMVarId) ← altMVarId'.introNP numGeneralized
+        pure altMVarId
+      else
+        pure altMVarId'
+      for fvarId in toClear do
+        altMVarId ← altMVarId.tryClear fvarId
+      altMVarId.withContext do
+        for (stx, fvar) in toTag do
+          Term.addLocalVarInfo stx (subst.get fvar)
+      let altMVarIds ← applyPreTac altMVarId
+      if !hasAlts then
+        -- User did not provide alternatives using `|`
+        setGoals <| (← getGoals) ++ altMVarIds
+      else if !altMVarIds.isEmpty then
+        logError m!"alternative '{altName}' has not been provided"
+        altMVarIds.forM fun mvarId => admitGoal mvarId
+
+  /-- Applies syntactic alternative to alternative goal. -/
+  applyAltStx tacSnaps altStxIdx altStx alt := withRef altStx do
+    let { name := altName, info, mvarId := altMVarId } := alt
+    -- also checks for unknown alternatives
+    let numFields ← getAltNumFields elimInfo altName
+    let altVars := getAltVars altStx
+    let numFieldsToName ← if altHasExplicitModifier altStx then pure numFields else getNumExplicitFields altMVarId numFields
+    if altVars.size > numFieldsToName then
+      logError m!"too many variable names provided at alternative '{altName}', #{altVars.size} provided, but #{numFieldsToName} expected"
+    let mut (fvarIds, altMVarId) ← altMVarId.introN numFields (altVars.toList.map getNameOfIdent') (useNamesForExplicitOnly := !altHasExplicitModifier altStx)
+    -- Delay adding the infos for the pattern LHS because we want them to nest
+    -- inside tacticInfo for the current alternative (in `evalAlt`)
+    let addInfo : TermElabM Unit := do
+      if (← getInfoState).enabled then
+        if let some declName := info.declName? then
+          addConstInfo (getAltNameStx altStx) declName
+        saveAltVarsInfo altMVarId altStx fvarIds
+    let unusedAlt := do
+      addInfo
+      if !isWildcard altStx then
+        throwError "alternative '{altName}' is not needed"
+    let some (altMVarId', subst) ← Cases.unifyEqs? numEqs altMVarId {}
+      | unusedAlt
+    altMVarId ← if info.provesMotive then
+      (_, altMVarId) ← altMVarId'.introNP numGeneralized
+      pure altMVarId
+    else
+      pure altMVarId'
+    for fvarId in toClear do
+      altMVarId ← altMVarId.tryClear fvarId
+    altMVarId.withContext do
+      for (stx, fvar) in toTag do
+        Term.addLocalVarInfo stx (subst.get fvar)
+    let altMVarIds ← applyPreTac altMVarId
+    if altMVarIds.isEmpty then
+      return (← unusedAlt)
+
+    -- select corresponding snapshot bundle for incrementality of this alternative
+    -- note that `tacSnaps[altStxIdx]?` is `none` if `tacSnap?` was `none` to begin with
+    withTheReader Term.Context ({ · with tacSnap? := tacSnaps[altStxIdx]? }) do
+    -- all previous alternatives have to be unchanged for reuse
+    Term.withNarrowedArgTacticReuse (stx := mkNullNode altStxs) (argIdx := altStxIdx) fun altStx => do
+    -- everything up to rhs has to be unchanged for reuse
+    Term.withNarrowedArgTacticReuse (stx := altStx) (argIdx := 2) fun _rhs => do
+    -- disable reuse if rhs is run multiple times
+    Term.withoutTacticIncrementality (altMVarIds.length != 1 || isWildcard altStx) do
+      for altMVarId' in altMVarIds do
+        evalAlt altMVarId' altStx addInfo
+
+  /-- Applies `induction .. with $preTac | ..`, if any, to an alternative goal. -/
   applyPreTac (mvarId : MVarId) : TacticM (List MVarId) :=
     if optPreTac.isNone then
       return [mvarId]
     else
-      evalTacticAt optPreTac[0] mvarId
+      -- disable incrementality for the pre-tactic to avoid non-monotonic progress reporting; it
+      -- would be possible to include a custom task around the pre-tac with an appropriate range in
+      -- the snapshot such that it is cached as well if it turns out that this is valuable
+      Term.withoutTacticIncrementality true do
+        evalTacticAt optPreTac[0] mvarId
 
 end ElimApp
 
@@ -420,8 +458,24 @@ Return an array containing its alternatives.
 private def getAltsOfInductionAlts (inductionAlts : Syntax) : Array Syntax :=
   inductionAlts[2].getArgs
 
-private def getAltsOfOptInductionAlts (optInductionAlts : Syntax) : Array Syntax :=
-  if optInductionAlts.isNone then #[] else getAltsOfInductionAlts optInductionAlts[0]
+/--
+Given `inductionAlts` of the form
+```
+syntax inductionAlts := "with " (tactic)? withPosition( (colGe inductionAlt)+)
+```
+runs `cont alts` where `alts` is an array containing all `inductionAlt`s while disabling incremental
+reuse if any other syntax changed.
+-/
+private def withAltsOfOptInductionAlts (optInductionAlts : Syntax)
+    (cont : Array Syntax → TacticM α) : TacticM α :=
+  Term.withNarrowedTacticReuse (stx := optInductionAlts) (fun optInductionAlts =>
+    if optInductionAlts.isNone then
+      -- if there are no alternatives, what to compare is irrelevant as there will be no reuse
+      (mkNullNode #[], mkNullNode #[])
+    else
+      -- `with` and tactic applied to all branches must be unchanged for reuse
+      (mkNullNode optInductionAlts[0].getArgs[:2], optInductionAlts[0].getArg 2))
+    (fun alts => cont alts.getArgs)
 
 private def getOptPreTacOfOptInductionAlts (optInductionAlts : Syntax) : Syntax :=
   if optInductionAlts.isNone then mkNullNode else optInductionAlts[0][1]
@@ -582,12 +636,11 @@ private def generalizeTargets (exprs : Array Expr) : TacticM (Array Expr) := do
   else
     return exprs
 
+builtin_initialize registerBuiltinIncrementalTactic ``Lean.Parser.Tactic.induction
 @[builtin_tactic Lean.Parser.Tactic.induction] def evalInduction : Tactic := fun stx =>
   match expandInduction? stx with
   | some stxNew => withMacroExpansion stx stxNew <| evalTactic stxNew
   | _ => focus do
-    let optInductionAlts := stx[4]
-    let alts := getAltsOfOptInductionAlts optInductionAlts
     let targets ← withMainContext <| stx[1].getSepArgs.mapM (elabTerm · none)
     let targets ← generalizeTargets targets
     let elimInfo ← withMainContext <| getElimNameInfo stx[2] targets (induction := true)
@@ -605,10 +658,15 @@ private def generalizeTargets (exprs : Array Expr) : TacticM (Array Expr) := do
           ElimApp.mkElimApp elimInfo targets tag
         trace[Elab.induction] "elimApp: {result.elimApp}"
         ElimApp.setMotiveArg mvarId result.motive targetFVarIds
-        let optPreTac := getOptPreTacOfOptInductionAlts optInductionAlts
-        mvarId.assign result.elimApp
-        ElimApp.evalAlts elimInfo result.alts optPreTac alts initInfo (numGeneralized := n) (toClear := targetFVarIds)
-        appendGoals result.others.toList
+        -- drill down into old and new syntax: allow reuse of an rhs only if everything before it is
+        -- unchanged
+        -- everything up to the alternatives must be unchanged for reuse
+        Term.withNarrowedArgTacticReuse (stx := stx) (argIdx := 4) fun optInductionAlts => do
+        withAltsOfOptInductionAlts optInductionAlts fun alts => do
+          let optPreTac := getOptPreTacOfOptInductionAlts optInductionAlts
+          mvarId.assign result.elimApp
+          ElimApp.evalAlts elimInfo result.alts optPreTac alts initInfo (numGeneralized := n) (toClear := targetFVarIds)
+          appendGoals result.others.toList
 where
   checkTargets (targets : Array Expr) : MetaM Unit := do
     let mut foundFVars : FVarIdSet := {}
@@ -650,15 +708,13 @@ def elabCasesTargets (targets : Array Syntax) : TacticM (Array Expr × Array (Id
     else
       return (args.map (·.expr), #[])
 
+builtin_initialize registerBuiltinIncrementalTactic ``Lean.Parser.Tactic.cases
 @[builtin_tactic Lean.Parser.Tactic.cases] def evalCases : Tactic := fun stx =>
   match expandCases? stx with
   | some stxNew => withMacroExpansion stx stxNew <| evalTactic stxNew
   | _ => focus do
     -- leading_parser nonReservedSymbol "cases " >> sepBy1 (group majorPremise) ", " >> usingRec >> optInductionAlts
     let (targets, toTag) ← elabCasesTargets stx[1].getSepArgs
-    let optInductionAlts := stx[3]
-    let optPreTac := getOptPreTacOfOptInductionAlts optInductionAlts
-    let alts :=  getAltsOfOptInductionAlts optInductionAlts
     let targetRef := stx[1]
     let elimInfo ← withMainContext <| getElimNameInfo stx[2] targets (induction := false)
     let mvarId ← getMainGoal
@@ -676,8 +732,14 @@ def elabCasesTargets (targets : Array Syntax) : TacticM (Array Expr × Array (Id
       mvarId.withContext do
         ElimApp.setMotiveArg mvarId elimArgs[elimInfo.motivePos]!.mvarId! targetsNew
         mvarId.assign result.elimApp
-        ElimApp.evalAlts elimInfo result.alts optPreTac alts initInfo
-          (numEqs := targets.size) (toClear := targetsNew) (toTag := toTag)
+        -- drill down into old and new syntax: allow reuse of an rhs only if everything before it is
+        -- unchanged
+        -- everything up to the alternatives must be unchanged for reuse
+        Term.withNarrowedArgTacticReuse (stx := stx) (argIdx := 3) fun optInductionAlts => do
+        withAltsOfOptInductionAlts optInductionAlts fun alts => do
+          let optPreTac := getOptPreTacOfOptInductionAlts optInductionAlts
+          ElimApp.evalAlts elimInfo result.alts optPreTac alts initInfo
+            (numEqs := targets.size) (toClear := targetsNew) (toTag := toTag)
 
 builtin_initialize
   registerTraceClass `Elab.cases

src/Lean/Elab/Tactic/Omega/OmegaM.lean

@@ -71,7 +71,7 @@ abbrev OmegaM := StateRefT Cache OmegaM'
 
 /-- Run a computation in the `OmegaM` monad, starting with no recorded atoms. -/
 def OmegaM.run (m : OmegaM α) (cfg : OmegaConfig) : MetaM α :=
-  m.run' HashMap.empty |>.run' {} { cfg } |>.run
+  m.run' HashMap.empty |>.run' {} { cfg } |>.run'
 
 /-- Retrieve the user-specified configuration options. -/
 def cfg : OmegaM OmegaConfig := do pure (← read).cfg

src/Lean/Elab/Term.lean

@@ -13,6 +13,7 @@ import Lean.Elab.Config
 import Lean.Elab.Level
 import Lean.Elab.DeclModifiers
 import Lean.Elab.PreDefinition.WF.TerminationHint
+import Lean.Language.Basic
 
 namespace Lean.Elab
 
@@ -112,6 +113,14 @@ structure State where
   letRecsToLift     : List LetRecToLift := []
   deriving Inhabited
 
+/--
+  Backtrackable state for the `TermElabM` monad.
+-/
+structure SavedState where
+  meta   : Meta.SavedState
+  «elab» : State
+  deriving Nonempty
+
 end Term
 
 namespace Tactic
@@ -152,6 +161,42 @@ structure Cache where
    post : PHashMap CacheKey Snapshot := {}
    deriving Inhabited
 
+section Snapshot
+open Language
+
+structure SavedState where
+  term   : Term.SavedState
+  tactic : State
+
+/-- State after finishing execution of a tactic. -/
+structure TacticFinished where
+  /-- Reusable state, if no fatal exception occurred. -/
+  state? : Option SavedState
+deriving Inhabited
+
+/-- Snapshot just before execution of a tactic. -/
+structure TacticParsedSnapshotData extends Language.Snapshot where
+  /-- Syntax tree of the tactic, stored and compared for incremental reuse. -/
+  stx      : Syntax
+  /-- Task for state after tactic execution. -/
+  finished : Task TacticFinished
+deriving Inhabited
+
+/-- State after execution of a single synchronous tactic step. -/
+inductive TacticParsedSnapshot where
+  | mk (data : TacticParsedSnapshotData) (next : Array (SnapshotTask TacticParsedSnapshot))
+deriving Inhabited
+abbrev TacticParsedSnapshot.data : TacticParsedSnapshot → TacticParsedSnapshotData
+  | .mk data _ => data
+/-- Potential, potentially parallel, follow-up tactic executions. -/
+-- In the first, non-parallel version, each task will depend on its predecessor
+abbrev TacticParsedSnapshot.next : TacticParsedSnapshot → Array (SnapshotTask TacticParsedSnapshot)
+  | .mk _ next => next
+partial instance : ToSnapshotTree TacticParsedSnapshot where
+  toSnapshotTree := go where
+    go := fun ⟨s, next⟩ => ⟨s.toSnapshot, next.map (·.map (sync := true) go)⟩
+
+end Snapshot
 end Tactic
 
 namespace Term
@@ -211,6 +256,13 @@ structure Context where
   /-- Cache for the `save` tactic. It is only `some` in the LSP server. -/
   tacticCache?     : Option (IO.Ref Tactic.Cache) := none
   /--
+  Snapshot for incremental processing of current tactic, if any.
+
+  Invariant: if the bundle's `old?` is set, then the state *up to the start* of the tactic is
+  unchanged, i.e. reuse is possible.
+  -/
+  tacSnap?         : Option (Language.SnapshotBundle Tactic.TacticParsedSnapshot) := none
+  /--
   If `true`, we store in the `Expr` the `Syntax` for recursive applications (i.e., applications
   of free variables tagged with `isAuxDecl`). We store the `Syntax` using `mkRecAppWithSyntax`.
   We use the `Syntax` object to produce better error messages at `Structural.lean` and `WF.lean`. -/
@@ -241,14 +293,6 @@ open Meta
 instance : Inhabited (TermElabM α) where
   default := throw default
 
-/--
-  Backtrackable state for the `TermElabM` monad.
--/
-structure SavedState where
-  meta   : Meta.SavedState
-  «elab» : State
-  deriving Nonempty
-
 protected def saveState : TermElabM SavedState :=
   return { meta := (← Meta.saveState), «elab» := (← get) }
 
@@ -261,18 +305,87 @@ def SavedState.restore (s : SavedState) (restoreInfo : Bool := false) : TermElab
   unless restoreInfo do
     setInfoState infoState
 
-/--
-Restores full state including sources for unique identifiers. Only intended for incremental reuse
-between elaboration runs, not for backtracking within a single run.
--/
-def SavedState.restoreFull (s : SavedState) : TermElabM Unit := do
-  s.meta.restoreFull
-  set s.elab
+@[specialize, inherit_doc Core.withRestoreOrSaveFull]
+def withRestoreOrSaveFull (reusableResult? : Option (α × SavedState))
+    (cont : TermElabM SavedState → TermElabM α) : TermElabM α := do
+  if let some (_, state) := reusableResult? then
+    set state.elab
+  let reusableResult? := reusableResult?.map (fun (val, state) => (val, state.meta))
+  controlAt MetaM fun runInBase =>
+    Meta.withRestoreOrSaveFull reusableResult? fun restore =>
+      runInBase <| cont (return { meta := (← restore), «elab» := (← get) })
 
 instance : MonadBacktrack SavedState TermElabM where
   saveState      := Term.saveState
   restoreState b := b.restore
 
+/--
+Manages reuse information for nested tactics by `split`ting given syntax into an outer and inner
+part. `act` is then run on the inner part but with reuse information adjusted as following:
+* If the old (from `tacSnap?`'s `SyntaxGuarded.stx`) and new (from `stx`) outer syntax are not
+  identical according to `Syntax.structRangeEq`, reuse is disabled.
+* Otherwise, the old syntax as stored in `tacSnap?` is updated to the old *inner* syntax.
+* In any case, we also use `withRef` on the inner syntax to avoid leakage of the outer syntax into
+  `act` via this route.
+
+For any tactic that participates in reuse, `withNarrowedTacticReuse` should be applied to the
+tactic's syntax and `act` should be used to do recursive tactic evaluation of nested parts.
+-/
+def withNarrowedTacticReuse [Monad m] [MonadExceptOf Exception m] [MonadWithReaderOf Context m]
+    [MonadOptions m] [MonadRef m] (split : Syntax → Syntax × Syntax) (act : Syntax → m α)
+    (stx : Syntax) : m α := do
+  let (outer, inner) := split stx
+  let opts ← getOptions
+  withTheReader Term.Context (fun ctx => { ctx with tacSnap? := ctx.tacSnap?.map fun tacSnap =>
+    { tacSnap with old? := tacSnap.old?.bind fun old => do
+      let (oldOuter, oldInner) := split old.stx
+      guard <| outer.structRangeEqWithTraceReuse opts oldOuter
+      return { old with stx := oldInner }
+    }
+  }) do
+    withRef inner do
+      act inner
+
+/--
+A variant of `withNarrowedTacticReuse` that uses `stx[argIdx]` as the inner syntax and all `stx`
+child nodes before that as the outer syntax, i.e. reuse is disabled if there was any change before
+`argIdx`.
+
+NOTE: child nodes after `argIdx` are not tested (which would almost always disable reuse as they are
+necessarily shifted by changes at `argIdx`) so it must be ensured that the result of `arg` does not
+depend on them (i.e. they should not be inspected beforehand).
+-/
+def withNarrowedArgTacticReuse [Monad m] [MonadExceptOf Exception m] [MonadWithReaderOf Context m]
+    [MonadOptions m] [MonadRef m] (argIdx : Nat) (act : Syntax → m α) (stx : Syntax) : m α :=
+  withNarrowedTacticReuse (fun stx => (mkNullNode stx.getArgs[:argIdx], stx[argIdx])) act stx
+
+/--
+Disables incremental tactic reuse *and* reporting for `act` if `cond` is true by setting `tacSnap?`
+to `none`. This should be done for tactic blocks that are run multiple times as otherwise the
+reported progress will jump back and forth (and partial reuse for these kinds of tact blocks is
+similarly questionable).
+-/
+def withoutTacticIncrementality [Monad m] [MonadWithReaderOf Context m] [MonadOptions m] [MonadRef m]
+    (cond : Bool) (act : m α) : m α := do
+  let opts ← getOptions
+  withTheReader Term.Context (fun ctx => { ctx with tacSnap? := ctx.tacSnap?.filter fun tacSnap => Id.run do
+    if let some old := tacSnap.old? then
+      if cond && opts.getBool `trace.Elab.reuse then
+        dbg_trace "reuse stopped: guard failed at {old.stx}"
+    return !cond
+  }) act
+
+/-- Disables incremental tactic reuse for `act` if `cond` is true. -/
+def withoutTacticReuse [Monad m] [MonadWithReaderOf Context m] [MonadOptions m] [MonadRef m]
+    (cond : Bool) (act : m α) : m α := do
+  let opts ← getOptions
+  withTheReader Term.Context (fun ctx => { ctx with tacSnap? := ctx.tacSnap?.map fun tacSnap =>
+    { tacSnap with old? := tacSnap.old?.filter fun old => Id.run do
+      if cond && opts.getBool `trace.Elab.reuse then
+        dbg_trace "reuse stopped: guard failed at {old.stx}"
+      return !cond }
+  }) act
+
 abbrev TermElabResult (α : Type) := EStateM.Result Exception SavedState α
 
 /--
@@ -1531,7 +1644,7 @@ partial def withAutoBoundImplicit (k : TermElabM α) : TermElabM α := do
           | ex => match isAutoBoundImplicitLocalException? ex with
             | some n =>
               -- Restore state, declare `n`, and try again
-              s.restore
+              s.restore (restoreInfo := true)
               withLocalDecl n .implicit (← mkFreshTypeMVar) fun x =>
                 withReader (fun ctx => { ctx with autoBoundImplicits := ctx.autoBoundImplicits.push x } ) do
                   loop (← saveState)

src/Lean/Elab/Util.lean

@@ -205,7 +205,7 @@ def logException [Monad m] [MonadLog m] [AddMessageContext m] [MonadOptions m] [
   match ex with
   | Exception.error ref msg => logErrorAt ref msg
   | Exception.internal id _ =>
-    unless isAbortExceptionId id do
+    unless isAbortExceptionId id || id == Core.interruptExceptionId do
       let name ← id.getName
       logError m!"internal exception: {name}"
 

src/Lean/Exception.lean

@@ -130,7 +130,7 @@ been defined yet.
 -/
 def Exception.isMaxRecDepth (ex : Exception) : Bool :=
   match ex with
-  | error _ (MessageData.ofFormat (Std.Format.text msg)) => msg == maxRecDepthErrorMessage
+  | error _ (MessageData.ofFormatWithInfos ⟨Std.Format.text msg, _⟩) => msg == maxRecDepthErrorMessage
   | _ => false
 
 /--

src/Lean/Language/Basic.lean

@@ -17,8 +17,13 @@ set_option linter.missingDocs true
 
 namespace Lean.Language
 
-/-- `MessageLog` with interactive diagnostics. -/
+/--
+`MessageLog` with interactive diagnostics.
+
+Can be created using `Diagnostics.empty` or `Diagnostics.ofMessageLog`.
+-/
 structure Snapshot.Diagnostics where
+  private mk ::
   /-- Non-interactive message log. -/
   msgLog : MessageLog
   /--
@@ -133,8 +138,7 @@ checking if we can reuse `old?` if set or else redoing the corresponding elabora
 case, we derive new bundles for nested snapshots, if any, and finally `resolve` `new` to the result.
 
 Note that failing to `resolve` a created promise will block the language server indefinitely!
-Corresponding `IO.Promise.new` calls should come with a "definitely resolved in ..." comment
-explaining how this is avoided in each case.
+We use `withAlwaysResolvedPromise`/`withAlwaysResolvedPromises` to ensure this doesn't happen.
 
 In the future, the 1-element history `old?` may be replaced with a global cache indexed by strong
 hashes but the promise will still need to be passed through the elaborator.
@@ -151,6 +155,36 @@ structure SnapshotBundle (α : Type) where
   -/
   new  : IO.Promise α
 
+/--
+Runs `act` with a newly created promise and finally resolves it to `default` if not done by `act`.
+
+Always resolving promises involved in the snapshot tree is important to avoid deadlocking the
+language server.
+-/
+def withAlwaysResolvedPromise [Monad m] [MonadLiftT BaseIO m] [MonadFinally m] [Inhabited α]
+    (act : IO.Promise α → m Unit) : m Unit := do
+  let p ← IO.Promise.new
+  try
+    act p
+  finally
+    p.resolve default
+
+/--
+Runs `act` with `count` newly created promises and finally resolves them to `default` if not done by
+`act`.
+
+Always resolving promises involved in the snapshot tree is important to avoid deadlocking the
+language server.
+-/
+def withAlwaysResolvedPromises [Monad m] [MonadLiftT BaseIO m] [MonadFinally m] [Inhabited α]
+    (count : Nat) (act : Array (IO.Promise α) → m Unit) : m Unit := do
+  let ps ← List.iota count |>.toArray.mapM fun _ => IO.Promise.new
+  try
+    act ps
+  finally
+    for p in ps do
+      p.resolve default
+
 /--
   Tree of snapshots where each snapshot comes with an array of asynchronous further subtrees. Used
   for asynchronously collecting information about the entirety of snapshots in the language server.

src/Lean/Language/Lean.lean

@@ -63,36 +63,61 @@ we remain at "go two commands up" at this point.
 
 Because of Lean's use of persistent data structures, incremental reuse of fully elaborated commands
 is easy because we can simply snapshot the entire state after each command and then restart
-elaboration using the stored state at the point of change. However, incrementality within
-elaboration of a single command such as between tactic steps is much harder because we cannot simply
-return from those points to the language processor in a way that we can later resume from there.
-Instead, we exchange the need for continuations with some limited mutability: by allocating an
-`IO.Promise` "cell" in the language processor, we can both pass it to the elaborator to eventually
-fill it using `Promise.resolve` as well as convert it to a `Task` that will wait on that resolution
-using `Promise.result` and return it as part of the command snapshot created by the language
-processor. The elaborator can then create new promises itself and store their `result` when
-resolving an outer promise to create an arbitrary tree of promise-backed snapshot tasks. Thus, we
-can enable incremental reporting and reuse inside the elaborator using the same snapshot tree data
-structures as outside without having to change the elaborator's control flow.
+elaboration using the stored state at the next command above the point of change. However,
+incrementality *within* elaboration of a single command such as between tactic steps is much harder
+because the existing control flow does not allow us to simply return from those points to the
+language processor in a way that we can later resume from there. Instead, we exchange the need for
+continuations with some limited mutability: by allocating an `IO.Promise` "cell" in the language
+processor, we can both pass it to the elaborator to eventually fill it using `Promise.resolve` as
+well as convert it to a `Task` that will wait on that resolution using `Promise.result` and return
+it as part of the command snapshot created by the language processor. The elaborator can then in
+turn create new promises itself and store their `result` when resolving an outer promise to create
+an arbitrary tree of promise-backed snapshot tasks. Thus, we can enable incremental reporting and
+reuse inside the elaborator using the same snapshot tree data structures as outside without having
+to change the elaborator's control flow.
 
 While ideally we would decide what can be reused during command elaboration using strong hashes over
-the state and inputs, currently we rely on simpler syntactic checks: if all the syntax inspected up
-to a certain point is unchanged, we can assume that the old state can be reused.  The central
-`SnapshotBundle` type passed inwards through the elaborator for this purpose combines the following
-data:
+the full state and inputs, currently we rely on simpler syntactic checks: if all the syntax
+inspected up to a certain point is unchanged, we can assume that the old state can be reused. The
+central `SnapshotBundle` type passed inwards through the elaborator for this purpose combines the
+following data:
 * the `IO.Promise` to be resolved to an elaborator snapshot (whose type depends on the specific
-  elaborator part we're in, e.g. `)
+  elaborator part we're in, e.g. `TacticParsedSnapshot`, `BodyProcessedSnapshot`)
 * if there was a previous run:
   * a `SnapshotTask` holding the corresponding snapshot of the run
   * the relevant `Syntax` of the previous run to be compared before any reuse
 
 Note that as we do not wait for the previous run to finish before starting to elaborate the next
-one, the `SnapshotTask` task may not be finished yet. Indeed, if we do find that we can reuse the
-contained state, we will want to explicitly wait for it instead of redoing the work. On the other
-hand, the `Syntax` is not surrounded by a task so that we can immediately access it for comparisons,
-even if the snapshot task may, eventually, give access to the same syntax tree.
+one, the old `SnapshotTask` task may not be finished yet. Indeed, if we do find that we can reuse
+the contained state because of a successful syntax comparison, we always want to explicitly wait for
+the task instead of redoing the work. On the other hand, the `Syntax` is not surrounded by a task so
+that we can immediately access it for comparisons, even if the snapshot task may, eventually, give
+access to the same syntax tree.
 
-TODO: tactic examples
+For the most part, inside an elaborator participating in incrementality, we just have to ensure that
+we stop forwarding the old run's data as soon as we notice a relevant difference between old and new
+syntax tree. For example, allowing incrementality inside the cdot tactic combinator is as simple as
+```
+builtin_initialize registerBuiltinIncrementalTactic ``cdot
+@[builtin_tactic cdot] def evalTacticCDot : Tactic := fun stx => do
+  ...
+  closeUsingOrAdmit do
+    -- save state before/after entering focus on `·`
+    ...
+    Term.withNarrowedArgTacticReuse (argIdx := 1) evalTactic stx
+```
+The `Term.withNarrowedArgTacticReuse` combinator will focus on the given argument of `stx`, which in
+this case is the nested tactic sequence, and run `evalTactic` on it. But crucially, it will first
+compare all preceding arguments, in this case the cdot token itself, with the old syntax in the
+current snapshot bundle, which in the case of tactics is stored in `Term.Context.tacSnap?`. Indeed
+it is important here to check if the cdot token is identical because its position has been saved in
+the info tree, so it would be bad if we later restored some old state that uses a different position
+for it even if everything else is unchanged.  If there is any mismatch, the bundle's old value is
+set to `none` in order to prevent reuse from this point on. Note that in any case we still want to
+forward the "new" promise in order to provide incremental reporting as well as to construct a
+snapshot tree for reuse in future document versions! Note also that we explicitly opted into
+incrementality using `registerBuiltinIncrementalTactic` as any tactic combinator not written with
+these concerns in mind would likely misbehave under incremental reuse.
 
 While it is generally true that we can provide incremental reporting even without reuse, we
 generally want to avoid that when it would be confusing/annoying, e.g. when a tactic block is run
@@ -101,12 +126,24 @@ purpose, we can disable both incremental modes using `Term.withoutTacticIncremen
 opted into incrementality because of other parts of the combinator. `induction` is an example of
 this because there are some induction alternatives that are run multiple times, so we disable all of
 incrementality for them.
+
+Using `induction` as a more complex example than `cdot` as it calls into `evalTactic` multiple
+times, here is a summary of what it has to do to implement incrementality:
+* `Narrow` down to the syntax of alternatives, disabling reuse if anything before them changed
+* allocate one new promise for each given alternative, immediately resolve passed promise to a new
+  snapshot tree node holding them so that the language server can wait on them
+* when executing an alternative,
+  * we put the corresponding promise into the context
+  * we disable reuse if anything in front of the contained tactic block has changed, including
+    previous alternatives
+  * we disable reuse *and reporting* if the tactic block is run multiple times, e.g. in the case of
+    a wildcard pattern
 -/
 
 set_option linter.missingDocs true
 
 namespace Lean.Language.Lean
-open Lean.Elab
+open Lean.Elab Command
 open Lean.Parser
 
 private def pushOpt (a? : Option α) (as : Array α) : Array α :=
@@ -121,12 +158,6 @@ register_builtin_option stderrAsMessages : Bool := {
   descr    := "(server) capture output to the Lean stderr channel (such as from `dbg_trace`) during elaboration of a command as a diagnostic message"
 }
 
-/-- Option for showing elaboration errors from partial syntax errors. -/
-register_builtin_option showPartialSyntaxErrors : Bool := {
-  defValue := false
-  descr    := "show elaboration errors from partial syntax trees (i.e. after parser recovery)"
-}
-
 /-! The hierarchy of Lean snapshot types -/
 
 /-- Snapshot after elaboration of the entire command. -/
@@ -165,7 +196,7 @@ deriving Nonempty
 abbrev CommandParsedSnapshot.data : CommandParsedSnapshot → CommandParsedSnapshotData
   | mk data _ => data
 /-- Next command, unless this is a terminal command. -/
-abbrev CommandParsedSnapshot.next? : CommandParsedSnapshot →
+abbrev CommandParsedSnapshot.nextCmdSnap? : CommandParsedSnapshot →
     Option (SnapshotTask CommandParsedSnapshot)
   | mk _ next? => next?
 partial instance : ToSnapshotTree CommandParsedSnapshot where
@@ -173,18 +204,7 @@ partial instance : ToSnapshotTree CommandParsedSnapshot where
     go s := ⟨s.data.toSnapshot,
       #[s.data.elabSnap.map (sync := true) toSnapshotTree,
         s.data.finishedSnap.map (sync := true) toSnapshotTree] |>
-        pushOpt (s.next?.map (·.map (sync := true) go))⟩
-
-
-/-- Cancels all significant computations from this snapshot onwards. -/
-partial def CommandParsedSnapshot.cancel (snap : CommandParsedSnapshot) : BaseIO Unit := do
-  -- This is the only relevant computation right now, everything else is promises
-  -- TODO: cancel additional elaboration tasks (which will be tricky with `DynamicSnapshot`) if we
-  -- add them without switching to implicit cancellation
-  snap.data.finishedSnap.cancel
-  if let some next := snap.next? then
-    -- recurse on next command (which may have been spawned just before we cancelled above)
-    let _ ← IO.mapTask (sync := true) (·.cancel) next.task
+        pushOpt (s.nextCmdSnap?.map (·.map (sync := true) go))⟩
 
 /-- State after successful importing. -/
 structure HeaderProcessedState where
@@ -218,6 +238,8 @@ structure HeaderParsedSnapshot extends Snapshot where
   /-- State after successful parsing. -/
   result? : Option HeaderParsedState
   isFatal := result?.isNone
+  /-- Cancellation token for interrupting processing of this run. -/
+  cancelTk? : Option IO.CancelToken
 
 instance : ToSnapshotTree HeaderParsedSnapshot where
   toSnapshotTree s := ⟨s.toSnapshot,
@@ -235,6 +257,10 @@ abbrev InitialSnapshot := HeaderParsedSnapshot
 structure LeanProcessingContext extends ProcessingContext where
   /-- Position of the first file difference if there was a previous invocation. -/
   firstDiffPos? : Option String.Pos
+  /-- Cancellation token of the previous invocation, if any. -/
+  oldCancelTk? : Option IO.CancelToken
+  /-- Cancellation token of the current run. -/
+  newCancelTk : IO.CancelToken
 
 /-- Monad transformer holding all relevant data for Lean processing. -/
 abbrev LeanProcessingT m := ReaderT LeanProcessingContext m
@@ -247,6 +273,18 @@ instance : MonadLift LeanProcessingM (LeanProcessingT IO) where
 instance : MonadLift (ProcessingT m) (LeanProcessingT m) where
   monadLift := fun act ctx => act ctx.toProcessingContext
 
+/--
+Embeds a `LeanProcessingM` action into `ProcessingM`, optionally using the old input string to speed
+up reuse analysis and supplying a cancellation token that should be triggered as soon as reuse is
+ruled out.
+-/
+def LeanProcessingM.run (act : LeanProcessingM α) (oldInputCtx? : Option InputContext)
+    (oldCancelTk? : Option IO.CancelToken := none) : ProcessingM α := do
+  -- compute position of syntactic change once
+  let firstDiffPos? := oldInputCtx?.map (·.input.firstDiffPos (← read).input)
+  let newCancelTk ← IO.CancelToken.new
+  ReaderT.adapt ({ · with firstDiffPos?, oldCancelTk?, newCancelTk }) act
+
 /--
 Returns true if there was a previous run and the given position is before any textual change
 compared to it.
@@ -291,8 +329,7 @@ General notes:
   state. As there is no cheap way to check whether the `Environment` is unchanged, i.e. *semantic*
   change detection is currently not possible, we must make sure to pass `none` as all follow-up
   "previous states" from the first *syntactic* change onwards.
-* We must make sure to use `CommandParsedSnapshot.cancel` on such tasks when discarding them, i.e.
-  when not passing them along in `old?`.
+* We must make sure to trigger `oldCancelTk?` as soon as discarding `old?`.
 * Control flow up to finding the last still-valid snapshot (which should be quick) is synchronous so
   as not to report this "fast forwarding" to the user as well as to make sure the next run sees all
   fast-forwarded snapshots without having to wait on tasks.
@@ -300,40 +337,47 @@ General notes:
 partial def process
     (setupImports : Syntax → ProcessingT IO (Except HeaderProcessedSnapshot SetupImportsResult))
     (old? : Option InitialSnapshot) : ProcessingM InitialSnapshot := do
-  -- compute position of syntactic change once
-  let firstDiffPos? := old?.map (·.ictx.input.firstDiffPos (← read).input)
-  ReaderT.adapt ({ · with firstDiffPos? }) do
-    parseHeader old?
+  parseHeader old? |>.run (old?.map (·.ictx)) (old?.bind (·.cancelTk?))
 where
   parseHeader (old? : Option HeaderParsedSnapshot) : LeanProcessingM HeaderParsedSnapshot := do
     let ctx ← read
     let ictx := ctx.toInputContext
-    let unchanged old :=
+    let unchanged old newParserState :=
       -- when header syntax is unchanged, reuse import processing task as is and continue with
       -- parsing the first command, synchronously if possible
+      -- NOTE: even if the syntax tree is functionally unchanged, the new parser state may still
+      -- have changed because of trailing whitespace and comments etc., so it is passed separately
+      -- from `old`
       if let some oldSuccess := old.result? then
-        return { old with ictx, result? := some { oldSuccess with
-          processedSnap := (← oldSuccess.processedSnap.bindIO (sync := true) fun oldProcessed => do
-            if let some oldProcSuccess := oldProcessed.result? then
-              -- also wait on old command parse snapshot as parsing is cheap and may allow for
-              -- elaboration reuse
-              oldProcSuccess.firstCmdSnap.bindIO (sync := true) fun oldCmd =>
-                return .pure { oldProcessed with result? := some { oldProcSuccess with
-                  firstCmdSnap := (← parseCmd oldCmd oldSuccess.parserState oldProcSuccess.cmdState ctx) } }
-            else
-              return .pure oldProcessed) } }
+        return {
+          ictx
+          stx := old.stx
+          diagnostics := old.diagnostics
+          cancelTk? := ctx.newCancelTk
+          result? := some { oldSuccess with
+            processedSnap := (← oldSuccess.processedSnap.bindIO (sync := true) fun oldProcessed => do
+              if let some oldProcSuccess := oldProcessed.result? then
+                -- also wait on old command parse snapshot as parsing is cheap and may allow for
+                -- elaboration reuse
+                oldProcSuccess.firstCmdSnap.bindIO (sync := true) fun oldCmd =>
+                  return .pure { oldProcessed with result? := some { oldProcSuccess with
+                    firstCmdSnap := (← parseCmd oldCmd newParserState oldProcSuccess.cmdState ctx) } }
+              else
+                return .pure oldProcessed) } }
       else return old
 
     -- fast path: if we have parsed the header successfully...
     if let some old := old? then
-      if let some (some processed) ← old.processedResult.get? then
-        -- ...and the edit location is after the next command (see note [Incremental Parsing])...
-        if let some nextCom ← processed.firstCmdSnap.get? then
-          if (← isBeforeEditPos nextCom.data.parserState.pos) then
-            -- ...go immediately to next snapshot
-            return (← unchanged old)
+      if let some oldSuccess := old.result? then
+        if let some (some processed) ← old.processedResult.get? then
+          -- ...and the edit location is after the next command (see note [Incremental Parsing])...
+          if let some nextCom ← processed.firstCmdSnap.get? then
+            if (← isBeforeEditPos nextCom.data.parserState.pos) then
+              -- ...go immediately to next snapshot
+              return (← unchanged old oldSuccess.parserState)
 
-    withHeaderExceptions ({ · with ictx, stx := .missing, result? := none }) do
+    withHeaderExceptions ({ · with
+        ictx, stx := .missing, result? := none, cancelTk? := none }) do
       -- parsing the header should be cheap enough to do synchronously
       let (stx, parserState, msgLog) ← Parser.parseHeader ictx
       if msgLog.hasErrors then
@@ -341,6 +385,7 @@ where
           ictx, stx
           diagnostics := (← Snapshot.Diagnostics.ofMessageLog msgLog)
           result? := none
+          cancelTk? := none
         }
 
       -- semi-fast path: go to next snapshot if syntax tree is unchanged AND we're still in front
@@ -351,14 +396,11 @@ where
       -- influence the range of error messages such as from a trailing `exact`
       if let some old := old? then
         if (← isBeforeEditPos parserState.pos) && old.stx == stx then
-          return (← unchanged old)
-        -- on first change, make sure to cancel all further old tasks
-        if let some oldSuccess := old.result? then
-          oldSuccess.processedSnap.cancel
-          let _ ← BaseIO.mapTask (t := oldSuccess.processedSnap.task) fun processed => do
-            if let some oldProcSuccess := processed.result? then
-              let _ ← BaseIO.mapTask (·.cancel) oldProcSuccess.firstCmdSnap.task
-
+          -- Here we must make sure to pass the *new* parser state; see NOTE in `unchanged`
+          return (← unchanged old parserState)
+        -- on first change, make sure to cancel old invocation
+        if let some tk := ctx.oldCancelTk? then
+          tk.set
       return {
         ictx, stx
         diagnostics := (← Snapshot.Diagnostics.ofMessageLog msgLog)
@@ -366,6 +408,7 @@ where
           parserState
           processedSnap := (← processHeader stx parserState)
         }
+        cancelTk? := ctx.newCancelTk
       }
 
   processHeader (stx : Syntax) (parserState : Parser.ModuleParserState) :
@@ -417,7 +460,7 @@ where
 
     -- check for cancellation, most likely during elaboration of previous command, before starting
     -- processing of next command
-    if (← IO.checkCanceled) then
+    if (← ctx.newCancelTk.isSet) then
       -- this is a bit ugly as we don't want to adjust our API with `Option`s just for cancellation
       -- (as no-one should look at this result in that case) but anything containing `Environment`
       -- is not `Inhabited`
@@ -428,22 +471,25 @@ where
         tacticCache := (← IO.mkRef {})
       }
 
-    let unchanged old : BaseIO CommandParsedSnapshot :=
+    let unchanged old newParserState : BaseIO CommandParsedSnapshot :=
       -- when syntax is unchanged, reuse command processing task as is
-      if let some oldNext := old.next? then
+      -- NOTE: even if the syntax tree is functionally unchanged, the new parser state may still
+      -- have changed because of trailing whitespace and comments etc., so it is passed separately
+      -- from `old`
+      if let some oldNext := old.nextCmdSnap? then
         return .mk (data := old.data)
           (nextCmdSnap? := (← old.data.finishedSnap.bindIO (sync := true) fun oldFinished =>
-              -- also wait on old command parse snapshot as parsing is cheap and may allow for
-              -- elaboration reuse
-              oldNext.bindIO (sync := true) fun oldNext => do
-                parseCmd oldNext old.data.parserState oldFinished.cmdState ctx))
+            -- also wait on old command parse snapshot as parsing is cheap and may allow for
+            -- elaboration reuse
+            oldNext.bindIO (sync := true) fun oldNext => do
+              parseCmd oldNext newParserState oldFinished.cmdState ctx))
       else return old  -- terminal command, we're done!
 
     -- fast path, do not even start new task for this snapshot
     if let some old := old? then
-      if let some nextCom ← old.next?.bindM (·.get?) then
+      if let some nextCom ← old.nextCmdSnap?.bindM (·.get?) then
         if (← isBeforeEditPos nextCom.data.parserState.pos) then
-          return .pure (← unchanged old)
+          return .pure (← unchanged old old.data.parserState)
 
     SnapshotTask.ofIO (some ⟨parserState.pos, ctx.input.endPos⟩) do
       let beginPos := parserState.pos
@@ -458,15 +504,19 @@ where
       -- semi-fast path
       if let some old := old? then
         if (← isBeforeEditPos parserState.pos ctx) && old.data.stx == stx then
-          return (← unchanged old)
-        -- on first change, make sure to cancel all further old tasks
-        old.cancel
+          -- Here we must make sure to pass the *new* parser state; see NOTE in `unchanged`
+          return (← unchanged old parserState)
+        -- on first change, make sure to cancel old invocation
+        -- TODO: pass token into incrementality-aware elaborators to improve reuse of still-valid,
+        -- still-running elaboration steps?
+        if let some tk := ctx.oldCancelTk? then
+          tk.set
 
       -- definitely resolved in `doElab` task
       let elabPromise ← IO.Promise.new
       let tacticCache ← old?.map (·.data.tacticCache) |>.getDM (IO.mkRef {})
       let finishedSnap ←
-        doElab stx cmdState msgLog.hasErrors beginPos
+        doElab stx cmdState beginPos
           { old? := old?.map fun old => ⟨old.data.stx, old.data.elabSnap⟩, new := elabPromise }
           tacticCache
           ctx
@@ -479,19 +529,24 @@ where
         diagnostics := (← Snapshot.Diagnostics.ofMessageLog msgLog)
         stx
         parserState
-        elabSnap := { range? := finishedSnap.range?, task := elabPromise.result }
+        elabSnap := { range? := stx.getRange?, task := elabPromise.result }
         finishedSnap
         tacticCache
       }
 
-  doElab (stx : Syntax) (cmdState : Command.State) (hasParseError : Bool) (beginPos : String.Pos)
+  doElab (stx : Syntax) (cmdState : Command.State) (beginPos : String.Pos)
       (snap : SnapshotBundle DynamicSnapshot) (tacticCache : IO.Ref Tactic.Cache) :
       LeanProcessingM (SnapshotTask CommandFinishedSnapshot) := do
     let ctx ← read
-
-    -- signature elaboration task; for now, does full elaboration
-    -- TODO: do tactic snapshots, reuse old state for them
-    SnapshotTask.ofIO (stx.getRange?.getD ⟨beginPos, beginPos⟩) do
+    -- (Try to) use last line of command as range for final snapshot task. This ensures we do not
+    -- retract the progress bar to a previous position in case the command support incremental
+    -- reporting but has significant work after resolving its last incremental promise, such as
+    -- final type checking; if it does not support incrementality, `elabSnap` constructed in
+    -- `parseCmd` and containing the entire range of the command will determine the reported
+    -- progress and be resolved effectively at the same time as this snapshot task, so `tailPos` is
+    -- irrelevant in this case.
+    let tailPos := stx.getTailPos? |>.getD beginPos
+    SnapshotTask.ofIO (some ⟨tailPos, tailPos⟩) do
       let scope := cmdState.scopes.head!
       let cmdStateRef ← IO.mkRef { cmdState with messages := .empty }
       /-
@@ -505,26 +560,18 @@ where
         cmdPos       := beginPos
         tacticCache? := some tacticCacheNew
         snap?        := some snap
+        cancelTk?    := some ctx.newCancelTk
       }
       let (output, _) ←
         IO.FS.withIsolatedStreams (isolateStderr := stderrAsMessages.get scope.opts) do
           liftM (m := BaseIO) do
-            Elab.Command.catchExceptions
+            withLoggingExceptions
               (getResetInfoTrees *> Elab.Command.elabCommandTopLevel stx)
               cmdCtx cmdStateRef
       let postNew := (← tacticCacheNew.get).post
       tacticCache.modify fun _ => { pre := postNew, post := {} }
       let cmdState ← cmdStateRef.get
       let mut messages := cmdState.messages
-      -- `stx.hasMissing` should imply `hasParseError`, but the latter should be cheaper to check in
-      -- general
-      if !showPartialSyntaxErrors.get cmdState.scopes[0]!.opts && hasParseError &&
-          stx.hasMissing then
-        -- discard elaboration errors, except for a few important and unlikely misleading ones, on
-        -- parse error
-        messages := ⟨messages.msgs.filter fun msg =>
-          msg.data.hasTag (fun tag => tag == `Elab.synthPlaceholder ||
-            tag == `Tactic.unsolvedGoals || (`_traceMsg).isSuffixOf tag)⟩
       if !output.isEmpty then
         messages := messages.add {
           fileName := ctx.fileName
@@ -541,13 +588,25 @@ where
         cmdState
       }
 
+/--
+Convenience function for tool uses of the language processor that skips header handling.
+-/
+def processCommands (inputCtx : Parser.InputContext) (parserState : Parser.ModuleParserState)
+    (commandState : Command.State)
+    (old? : Option (Parser.InputContext × CommandParsedSnapshot) := none) :
+    BaseIO (SnapshotTask CommandParsedSnapshot) := do
+  process.parseCmd (old?.map (·.2)) parserState commandState
+    |>.run (old?.map (·.1))
+    |>.run { inputCtx with }
+
+
 /-- Waits for and returns final environment, if importing was successful. -/
 partial def waitForFinalEnv? (snap : InitialSnapshot) : Option Environment := do
   let snap ← snap.result?
   let snap ← snap.processedSnap.get.result?
   goCmd snap.firstCmdSnap.get
 where goCmd snap :=
-  if let some next := snap.next? then
+  if let some next := snap.nextCmdSnap? then
     goCmd next.get
   else
     snap.data.finishedSnap.get.cmdState.env

src/Lean/Message.lean

@@ -39,13 +39,6 @@ structure NamingContext where
   currNamespace : Name
   openDecls : List OpenDecl
 
-/-- Lazily formatted text to be used in `MessageData`. -/
-structure PPFormat where
-  /-- Pretty-prints text using surrounding context, if any. -/
-  pp : Option PPContext → IO FormatWithInfos
-  /-- Searches for synthetic sorries in original input. Used to filter out certain messages. -/
-  hasSyntheticSorry : MetavarContext → Bool := fun _ => false
-
 structure TraceData where
   /-- Trace class, e.g. `Elab.step`. -/
   cls       : Name
@@ -60,10 +53,9 @@ structure TraceData where
 
 /-- Structured message data. We use it for reporting errors, trace messages, etc. -/
 inductive MessageData where
-  /-- Eagerly formatted text. We inspect this in various hacks, so it is not immediately subsumed by `ofPPFormat`. -/
-  | ofFormat          : Format → MessageData
-  /-- Lazily formatted text. -/
-  | ofPPFormat        : PPFormat → MessageData
+  /-- Eagerly formatted text with info annotations.
+  This constructor is inspected in various hacks. -/
+  | ofFormatWithInfos : FormatWithInfos → MessageData
   | ofGoal            : MVarId → MessageData
   /-- `withContext ctx d` specifies the pretty printing context `(env, mctx, lctx, opts)` for the nested expressions in `d`. -/
   | withContext       : MessageDataContext → MessageData → MessageData
@@ -78,12 +70,45 @@ inductive MessageData where
     Example: an inspector that tries to find "definitional equality failures" may look for the tag "DefEqFailure". -/
   | tagged            : Name → MessageData → MessageData
   | trace (data : TraceData) (msg : MessageData) (children : Array MessageData)
-  deriving Inhabited
+  /-- A lazy message.
+  The provided thunk will not be run until it is about to be displayed.
+  This can save computation in cases where the message may never be seen,
+  e.g. when nested inside a collapsed trace.
+
+  The `Dynamic` value is expected to be a `MessageData`,
+  which is a workaround for the positivity restriction.
+
+  If the thunked message is produced for a term that contains a synthetic sorry,
+  `hasSyntheticSorry` should return `true`.
+  This is used to filter out certain messages. -/
+  | ofLazy (f : Option PPContext → IO Dynamic) (hasSyntheticSorry : MetavarContext → Bool)
+  deriving Inhabited, TypeName
 
 namespace MessageData
 
+/-- Eagerly formatted text. -/
+def ofFormat (fmt : Format) : MessageData := .ofFormatWithInfos ⟨fmt, .empty⟩
+
+/--
+Lazy message data production, with access to the context as given by
+a surrounding `MessageData.withContext` (which is expected to exist).
+-/
+def lazy (f : PPContext → IO MessageData)
+    (hasSyntheticSorry : MetavarContext → Bool := fun _ => false) : MessageData :=
+  .ofLazy (hasSyntheticSorry := hasSyntheticSorry) fun ctx? => do
+    let msg ← match ctx? with
+      | .none => pure (.ofFormat "(invalid MessageData.lazy, missing context)")
+      | .some ctx => f ctx
+    return Dynamic.mk msg
+
 variable (p : Name → Bool) in
-/-- Returns true when the message contains a `MessageData.tagged tag ..` constructor where `p tag` is true. -/
+/-- Returns true when the message contains a `MessageData.tagged tag ..` constructor where `p tag`
+is true.
+
+This does not descend into lazily generated subtress (`.ofLazy`); message tags
+of interest (like those added by `logLinter`) are expected to be near the root
+of the `MessageData`, and not hidden inside `.ofLazy`.
+-/
 partial def hasTag : MessageData → Bool
   | withContext _ msg       => hasTag msg
   | withNamingContext _ msg => hasTag msg
@@ -106,26 +131,14 @@ def mkPPContext (nCtx : NamingContext) (ctx : MessageDataContext) : PPContext :=
 def ofSyntax (stx : Syntax) : MessageData :=
   -- discard leading/trailing whitespace
   let stx := stx.copyHeadTailInfoFrom .missing
-  .ofPPFormat {
-    pp := fun
-      | some ctx => ppTerm ctx ⟨stx⟩  -- HACK: might not be a term
-      | none     => return stx.formatStx
-  }
+  .lazy fun ctx => ofFormat <$> ppTerm ctx ⟨stx⟩ -- HACK: might not be a term
 
 def ofExpr (e : Expr) : MessageData :=
-  .ofPPFormat {
-    pp := fun
-      | some ctx => ppExprWithInfos ctx e
-      | none     => return format (toString e)
-    hasSyntheticSorry := (instantiateMVarsCore · e |>.1.hasSyntheticSorry)
-  }
+  .lazy (fun ctx => ofFormatWithInfos <$> ppExprWithInfos ctx e)
+        (fun mctx => instantiateMVarsCore mctx e |>.1.hasSyntheticSorry)
 
 def ofLevel (l : Level) : MessageData :=
-  .ofPPFormat {
-    pp := fun
-      | some ctx => ppLevel ctx l
-      | none => return format l
-  }
+  .lazy fun ctx => ofFormat <$> ppLevel ctx l
 
 def ofName (n : Name) : MessageData := ofFormat (format n)
 
@@ -133,7 +146,7 @@ partial def hasSyntheticSorry (msg : MessageData) : Bool :=
   visit none msg
 where
   visit (mctx? : Option MetavarContext) : MessageData → Bool
-  | ofPPFormat f            => f.hasSyntheticSorry (mctx?.getD {})
+  | ofLazy _ f              => f (mctx?.getD {})
   | withContext ctx msg     => visit ctx.mctx msg
   | withNamingContext _ msg => visit mctx? msg
   | nest _ msg              => visit mctx? msg
@@ -144,8 +157,7 @@ where
   | _                       => false
 
 partial def formatAux : NamingContext → Option MessageDataContext → MessageData → IO Format
-  | _,    _,         ofFormat fmt             => return fmt
-  | nCtx, ctx?,      ofPPFormat f             => (·.fmt) <$> f.pp (ctx?.map (mkPPContext nCtx))
+  | _, _,            ofFormatWithInfos fmt    => return fmt.1
   | _,    none,      ofGoal mvarId            => return "goal " ++ format (mkMVar mvarId)
   | nCtx, some ctx,  ofGoal mvarId            => ppGoal (mkPPContext nCtx ctx) mvarId
   | nCtx, _,         withContext ctx d        => formatAux nCtx ctx d
@@ -161,6 +173,11 @@ partial def formatAux : NamingContext → Option MessageDataContext → MessageD
     msg := f!"{msg} {(← formatAux nCtx ctx header).nest 2}"
     let children ← children.mapM (formatAux nCtx ctx)
     return .nest 2 (.joinSep (msg::children.toList) "\n")
+  | nCtx, ctx?,      ofLazy pp _             => do
+    let dyn ← pp (ctx?.map (mkPPContext nCtx))
+    let some msg := dyn.get? MessageData
+      | panic! s!"MessageData.ofLazy: expected MessageData in Dynamic, got {dyn.typeName}"
+    formatAux nCtx ctx? msg
 
 protected def format (msgData : MessageData) : IO Format :=
   formatAux { currNamespace := Name.anonymous, openDecls := [] } none msgData
@@ -281,47 +298,69 @@ protected def toJson (msg : Message) : IO Json := do
 
 end Message
 
-/-- A persistent array of messages. -/
+/--
+A persistent array of messages.
+
+In the Lean elaborator, we use a fresh message log per command but may also report diagnostics at
+various points inside a command, which will empty `unreported` and updated `hadErrors` accordingly
+(see `CoreM.getAndEmptyMessageLog`).
+-/
 structure MessageLog where
-  msgs : PersistentArray Message := {}
+  /--
+  If true, there was an error in the log previously that has already been reported to the user and
+  removed from the log. Thus we say that in the current context (usually the current command), we
+  "have errors" if either this flag is set or there is an error in `msgs`; see
+  `MessageLog.hasErrors`. If we have errors, we suppress some error messages that are often the
+  result of a previous error.
+  -/
+  /-
+  Design note: We considered introducing a `hasErrors` field instead that already includes the
+  presence of errors in `msgs` but this would not be compatible with e.g.
+  `MessageLog.errorsToWarnings`.
+  -/
+  hadErrors : Bool := false
+  /-- The list of messages not already reported, in insertion order. -/
+  unreported : PersistentArray Message := {}
   deriving Inhabited
 
 namespace MessageLog
-def empty : MessageLog := ⟨{}⟩
+def empty : MessageLog := {}
 
-def isEmpty (log : MessageLog) : Bool :=
-  log.msgs.isEmpty
+@[deprecated "renamed to `unreported`; direct access should in general be avoided in favor of \
+using `MessageLog.toList/toArray`"]
+def msgs : MessageLog → PersistentArray Message := unreported
+
+def hasUnreported (log : MessageLog) : Bool :=
+  !log.unreported.isEmpty
 
 def add (msg : Message) (log : MessageLog) : MessageLog :=
-  ⟨log.msgs.push msg⟩
+  { log with unreported := log.unreported.push msg }
 
 protected def append (l₁ l₂ : MessageLog) : MessageLog :=
-  ⟨l₁.msgs ++ l₂.msgs⟩
+  { hadErrors := l₁.hadErrors || l₂.hadErrors, unreported := l₁.unreported ++ l₂.unreported }
 
 instance : Append MessageLog :=
   ⟨MessageLog.append⟩
 
 def hasErrors (log : MessageLog) : Bool :=
-  log.msgs.any fun m => match m.severity with
-    | MessageSeverity.error => true
-    | _                     => false
+  log.hadErrors || log.unreported.any (·.severity matches .error)
 
 def errorsToWarnings (log : MessageLog) : MessageLog :=
-  { msgs := log.msgs.map (fun m => match m.severity with | MessageSeverity.error => { m with severity := MessageSeverity.warning } | _ => m) }
+  { unreported := log.unreported.map (fun m => match m.severity with | MessageSeverity.error => { m with severity := MessageSeverity.warning } | _ => m) }
 
 def getInfoMessages (log : MessageLog) : MessageLog :=
-  { msgs := log.msgs.filter fun m => match m.severity with | MessageSeverity.information => true | _ => false }
+  { unreported := log.unreported.filter fun m => match m.severity with | MessageSeverity.information => true | _ => false }
 
 def forM {m : Type → Type} [Monad m] (log : MessageLog) (f : Message → m Unit) : m Unit :=
-  log.msgs.forM f
+  log.unreported.forM f
 
-/-- Converts the log to a list, oldest message first. -/
+/-- Converts the unreported messages to a list, oldest message first. -/
 def toList (log : MessageLog) : List Message :=
-  log.msgs.toList
+  log.unreported.toList
 
-/-- Converts the log to an array, oldest message first. -/
+/-- Converts the unreported messages to an array, oldest message first. -/
 def toArray (log : MessageLog) : Array Message :=
-  log.msgs.toArray
+  log.unreported.toArray
 
 end MessageLog
 

src/Lean/Meta/Basic.lean

@@ -304,7 +304,7 @@ structure State where
   Backtrackable state for the `MetaM` monad.
 -/
 structure SavedState where
-  core        : Core.State
+  core        : Core.SavedState
   meta        : State
   deriving Nonempty
 
@@ -410,20 +410,22 @@ instance : AddMessageContext MetaM where
   addMessageContext := addMessageContextFull
 
 protected def saveState : MetaM SavedState :=
-  return { core := (← getThe Core.State), meta := (← get) }
+  return { core := (← Core.saveState), meta := (← get) }
 
 /-- Restore backtrackable parts of the state. -/
 def SavedState.restore (b : SavedState) : MetaM Unit := do
-  Core.restore b.core
+  b.core.restore
   modify fun s => { s with mctx := b.meta.mctx, zetaDeltaFVarIds := b.meta.zetaDeltaFVarIds, postponed := b.meta.postponed }
 
-/--
-Restores full state including sources for unique identifiers. Only intended for incremental reuse
-between elaboration runs, not for backtracking within a single run.
--/
-def SavedState.restoreFull (b : SavedState) : MetaM Unit := do
-  Core.restoreFull b.core
-  set b.meta
+@[specialize, inherit_doc Core.withRestoreOrSaveFull]
+def withRestoreOrSaveFull (reusableResult? : Option (α × SavedState))
+    (cont : MetaM SavedState → MetaM α) : MetaM α := do
+  if let some (_, state) := reusableResult? then
+    set state.meta
+  let reusableResult? := reusableResult?.map (fun (val, state) => (val, state.core))
+  controlAt CoreM fun runInCoreM =>
+    Core.withRestoreOrSaveFull reusableResult? fun restore =>
+      runInCoreM <| cont (return { core := (← restore), meta := (← get) })
 
 instance : MonadBacktrack SavedState MetaM where
   saveState      := Meta.saveState
@@ -720,15 +722,18 @@ def isReadOnlyExprMVar (mvarId : MVarId) : MetaM Bool := do
   mvarId.isReadOnly
 
 /--
-Return true if `mvarId.isReadOnly` return true or if `mvarId` is a synthetic opaque metavariable.
+Returns true if `mvarId.isReadOnly` returns true or if `mvarId` is a synthetic opaque metavariable.
 
 Recall `isDefEq` will not assign a value to `mvarId` if `mvarId.isReadOnlyOrSyntheticOpaque`.
 -/
 def _root_.Lean.MVarId.isReadOnlyOrSyntheticOpaque (mvarId : MVarId) : MetaM Bool := do
   let mvarDecl ← mvarId.getDecl
-  match mvarDecl.kind with
-  | MetavarKind.syntheticOpaque => return !(← getConfig).assignSyntheticOpaque
-  | _ => return mvarDecl.depth != (← getMCtx).depth
+  if mvarDecl.depth != (← getMCtx).depth then
+    return true
+  else
+    match mvarDecl.kind with
+    | MetavarKind.syntheticOpaque => return !(← getConfig).assignSyntheticOpaque
+    | _ => return false
 
 @[deprecated MVarId.isReadOnlyOrSyntheticOpaque (since := "2022-07-15")]
 def isReadOnlyOrSyntheticOpaqueExprMVar (mvarId : MVarId) : MetaM Bool := do
@@ -1457,7 +1462,9 @@ def withLocalInstancesImp (decls : List LocalDecl) (k : MetaM α) : MetaM α :=
   for decl in decls do
     unless decl.isImplementationDetail do
       if let some className ← isClass? decl.type then
-        localInsts := localInsts.push { className, fvar := decl.toExpr }
+        -- Ensure we don't add the same local instance multiple times.
+        unless localInsts.any fun localInst => localInst.fvar.fvarId! == decl.fvarId do
+          localInsts := localInsts.push { className, fvar := decl.toExpr }
   if localInsts.size == size then
     k
   else

src/Lean/Meta/Canonicalizer.lean

@@ -63,8 +63,11 @@ abbrev CanonM := ReaderT TransparencyMode $ StateRefT State MetaM
 The definitionally equality tests are performed using the given transparency mode.
 We claim `TransparencyMode.instances` is a good setting for most applications.
 -/
-def CanonM.run (x : CanonM α) (transparency := TransparencyMode.instances) : MetaM α :=
-  StateRefT'.run' (x transparency) {}
+def CanonM.run' (x : CanonM α) (transparency := TransparencyMode.instances) (s : State := {}) : MetaM α :=
+  StateRefT'.run' (x transparency) s
+
+def CanonM.run (x : CanonM α) (transparency := TransparencyMode.instances) (s : State := {}) : MetaM (α × State) :=
+  StateRefT'.run (x transparency) s
 
 private partial def mkKey (e : Expr) : CanonM UInt64 := do
   if let some hash := unsafe (← get).cache.find? { e } then

src/Lean/Meta/CtorRecognizer.lean

@@ -14,13 +14,11 @@ private def getConstructorVal? (env : Environment) (ctorName : Name) : Option Co
   | some (.ctorInfo v) => v
   | _                  => none
 
-
 /--
-If `e` is a constructor application or a builtin literal defeq to a constructor application,
+If `e` is a constructor application,
 then return the corresponding `ConstructorVal`.
 -/
-def isConstructorApp? (e : Expr) : MetaM (Option ConstructorVal) := do
-  let e ← litToCtor e
+def isConstructorAppCore? (e : Expr) : MetaM (Option ConstructorVal) := do
   let .const n _ := e.getAppFn | return none
   let some v := getConstructorVal? (← getEnv) n | return none
   if v.numParams + v.numFields == e.getAppNumArgs then
@@ -28,6 +26,13 @@ def isConstructorApp? (e : Expr) : MetaM (Option ConstructorVal) := do
   else
     return none
 
+/--
+If `e` is a constructor application or a builtin literal defeq to a constructor application,
+then return the corresponding `ConstructorVal`.
+-/
+def isConstructorApp? (e : Expr) : MetaM (Option ConstructorVal) := do
+  isConstructorAppCore? (← litToCtor e)
+
 /--
 Similar to `isConstructorApp?`, but uses `whnf`.
 -/

src/Lean/Meta/ExprDefEq.lean

@@ -740,58 +740,59 @@ mutual
     if mvarId == ctx.mvarId then
       traceM `Meta.isDefEq.assign.occursCheck <| addAssignmentInfo "occurs check failed"
       throwCheckAssignmentFailure
-    else match (← getExprMVarAssignment? mvarId) with
-      | some v => check v
-      | none   =>
-        match (← mvarId.findDecl?) with
-        | none          => throwUnknownMVar mvarId
-        | some mvarDecl =>
-          if ctx.hasCtxLocals then
-            throwCheckAssignmentFailure -- It is not a pattern, then we fail and fall back to FO unification
-          else if mvarDecl.lctx.isSubPrefixOf ctx.mvarDecl.lctx ctx.fvars then
-            /- The local context of `mvar` - free variables being abstracted is a subprefix of the metavariable being assigned.
-               We "subtract" variables being abstracted because we use `elimMVarDeps` -/
-            pure mvar
-          else if mvarDecl.depth != (← getMCtx).depth || mvarDecl.kind.isSyntheticOpaque then
-            traceM `Meta.isDefEq.assign.readOnlyMVarWithBiggerLCtx <| addAssignmentInfo (mkMVar mvarId)
-            throwCheckAssignmentFailure
-          else
-            let ctxMeta ← readThe Meta.Context
-            if ctxMeta.config.ctxApprox && ctx.mvarDecl.lctx.isSubPrefixOf mvarDecl.lctx then
-              /- Create an auxiliary metavariable with a smaller context and "checked" type.
-                 Note that `mvarType` may be different from `mvarDecl.type`. Example: `mvarType` contains
-                 a metavariable that we also need to reduce the context.
+    if let some v ← getExprMVarAssignment? mvarId then
+      return (← check v)
+    let some mvarDecl ← mvarId.findDecl?
+      | throwUnknownMVar mvarId
+    if ctx.hasCtxLocals then
+      throwCheckAssignmentFailure -- It is not a pattern, then we fail and fall back to FO unification
+    if let some d ← getDelayedMVarAssignment? mvarId then
+      -- we must perform occurs-check at `d.mvarIdPending`
+      unless (← occursCheck ctx.mvarId (mkMVar d.mvarIdPending)) do
+        traceM `Meta.isDefEq.assign.occursCheck <| addAssignmentInfo "occurs check failed"
+        throwCheckAssignmentFailure
+    if mvarDecl.lctx.isSubPrefixOf ctx.mvarDecl.lctx ctx.fvars then
+      /- The local context of `mvar` - free variables being abstracted is a subprefix of the metavariable being assigned.
+         We "subtract" variables being abstracted because we use `elimMVarDeps` -/
+      return mvar
+    if mvarDecl.depth != (← getMCtx).depth || mvarDecl.kind.isSyntheticOpaque then
+      traceM `Meta.isDefEq.assign.readOnlyMVarWithBiggerLCtx <| addAssignmentInfo (mkMVar mvarId)
+      throwCheckAssignmentFailure
+    let ctxMeta ← readThe Meta.Context
+    unless ctxMeta.config.ctxApprox && ctx.mvarDecl.lctx.isSubPrefixOf mvarDecl.lctx do
+      traceM `Meta.isDefEq.assign.readOnlyMVarWithBiggerLCtx <| addAssignmentInfo (mkMVar mvarId)
+      throwCheckAssignmentFailure
+    /- Create an auxiliary metavariable with a smaller context and "checked" type.
+       Note that `mvarType` may be different from `mvarDecl.type`. Example: `mvarType` contains
+       a metavariable that we also need to reduce the context.
 
-                 We remove from `ctx.mvarDecl.lctx` any variable that is not in `mvarDecl.lctx`
-                 or in `ctx.fvars`. We don't need to remove the ones in `ctx.fvars` because
-                 `elimMVarDeps` will take care of them.
+       We remove from `ctx.mvarDecl.lctx` any variable that is not in `mvarDecl.lctx`
+       or in `ctx.fvars`. We don't need to remove the ones in `ctx.fvars` because
+       `elimMVarDeps` will take care of them.
 
-                 First, we collect `toErase` the variables that need to be erased.
-                 Notat that if a variable is `ctx.fvars`, but it depends on variable at `toErase`,
-                 we must also erase it.
-              -/
-              let toErase ← mvarDecl.lctx.foldlM (init := #[]) fun toErase localDecl => do
-                if ctx.mvarDecl.lctx.contains localDecl.fvarId then
-                  return toErase
-                else if ctx.fvars.any fun fvar => fvar.fvarId! == localDecl.fvarId then
-                  if (← findLocalDeclDependsOn localDecl fun fvarId => toErase.contains fvarId) then
-                    -- localDecl depends on a variable that will be erased. So, we must add it to `toErase` too
-                    return toErase.push localDecl.fvarId
-                  else
-                    return toErase
-                else
-                  return toErase.push localDecl.fvarId
-              let lctx := toErase.foldl (init := mvarDecl.lctx) fun lctx toEraseFVar =>
-                lctx.erase toEraseFVar
-              /- Compute new set of local instances. -/
-              let localInsts := mvarDecl.localInstances.filter fun localInst => !toErase.contains localInst.fvar.fvarId!
-              let mvarType ← check mvarDecl.type
-              let newMVar ← mkAuxMVar lctx localInsts mvarType mvarDecl.numScopeArgs
-              mvarId.assign newMVar
-              pure newMVar
-            else
-              traceM `Meta.isDefEq.assign.readOnlyMVarWithBiggerLCtx <| addAssignmentInfo (mkMVar mvarId)
-              throwCheckAssignmentFailure
+       First, we collect `toErase` the variables that need to be erased.
+       Notat that if a variable is `ctx.fvars`, but it depends on variable at `toErase`,
+       we must also erase it.
+    -/
+    let toErase ← mvarDecl.lctx.foldlM (init := #[]) fun toErase localDecl => do
+      if ctx.mvarDecl.lctx.contains localDecl.fvarId then
+        return toErase
+      else if ctx.fvars.any fun fvar => fvar.fvarId! == localDecl.fvarId then
+        if (← findLocalDeclDependsOn localDecl fun fvarId => toErase.contains fvarId) then
+          -- localDecl depends on a variable that will be erased. So, we must add it to `toErase` too
+          return toErase.push localDecl.fvarId
+        else
+          return toErase
+      else
+        return toErase.push localDecl.fvarId
+    let lctx := toErase.foldl (init := mvarDecl.lctx) fun lctx toEraseFVar =>
+      lctx.erase toEraseFVar
+    /- Compute new set of local instances. -/
+    let localInsts := mvarDecl.localInstances.filter fun localInst => !toErase.contains localInst.fvar.fvarId!
+    let mvarType ← check mvarDecl.type
+    let newMVar ← mkAuxMVar lctx localInsts mvarType mvarDecl.numScopeArgs
+    mvarId.assign newMVar
+    return newMVar
 
   /--
     Auxiliary function used to "fix" subterms of the form `?m x_1 ... x_n` where `x_i`s are free variables,
@@ -802,12 +803,10 @@ mutual
   partial def assignToConstFun (mvar : Expr) (numArgs : Nat) (newMVar : Expr) : MetaM Bool := do
     let mvarType ← inferType mvar
     forallBoundedTelescope mvarType numArgs fun xs _ => do
-      if xs.size != numArgs then pure false
-      else
-        let some v ← mkLambdaFVarsWithLetDeps xs newMVar | return false
-        match (← checkAssignmentAux mvar.mvarId! #[] false v) with
-        | some v => checkTypesAndAssign mvar v
-        | none   => return false
+      if xs.size != numArgs then return false
+      let some v ← mkLambdaFVarsWithLetDeps xs newMVar | return false
+      let some v ← checkAssignmentAux mvar.mvarId! #[] false v | return false
+      checkTypesAndAssign mvar v
 
   -- See checkAssignment
   partial def checkAssignmentAux (mvarId : MVarId) (fvars : Array Expr) (hasCtxLocals : Bool) (v : Expr) : MetaM (Option Expr) := do
@@ -825,19 +824,18 @@ mutual
           (fun ex => do
             if !f.isMVar then
               throw ex
-            else if (← f.mvarId!.isDelayedAssigned) then
+            if (← f.mvarId!.isDelayedAssigned) then
               throw ex
+            let eType ← inferType e
+            let mvarType ← check eType
+            /- Create an auxiliary metavariable with a smaller context and "checked" type, assign `?f := fun _ => ?newMVar`
+               Note that `mvarType` may be different from `eType`. -/
+            let ctx ← read
+            let newMVar ← mkAuxMVar ctx.mvarDecl.lctx ctx.mvarDecl.localInstances mvarType
+            if (← assignToConstFun f args.size newMVar) then
+              pure newMVar
             else
-              let eType ← inferType e
-              let mvarType ← check eType
-              /- Create an auxiliary metavariable with a smaller context and "checked" type, assign `?f := fun _ => ?newMVar`
-                    Note that `mvarType` may be different from `eType`. -/
-              let ctx ← read
-              let newMVar ← mkAuxMVar ctx.mvarDecl.lctx ctx.mvarDecl.localInstances mvarType
-              if (← assignToConstFun f args.size newMVar) then
-                pure newMVar
-              else
-                throw ex)
+              throw ex)
       else
         let f ← check f
         let args ← args.mapM check
@@ -900,38 +898,33 @@ namespace CheckAssignmentQuick
 partial def check
     (hasCtxLocals : Bool)
     (mctx : MetavarContext) (lctx : LocalContext) (mvarDecl : MetavarDecl) (mvarId : MVarId) (fvars : Array Expr) (e : Expr) : Bool :=
-  let rec visit (e : Expr) : Bool :=
+  let rec visit (e : Expr) : Bool := Id.run do
     if !e.hasExprMVar && !e.hasFVar then
-      true
-    else match e with
+      return true
+    match e with
     | .mdata _ b       => visit b
     | .proj _ _ s      => visit s
     | .app f a         => visit f && visit a
     | .lam _ d b _     => visit d && visit b
     | .forallE _ d b _ => visit d && visit b
     | .letE _ t v b _  => visit t && visit v && visit b
-    | .bvar ..         => true
-    | .sort ..         => true
-    | .const ..        => true
-    | .lit ..          => true
+    | .bvar .. | .sort .. | .const ..
+    | .lit .. => return true
     | .fvar fvarId ..  =>
-      if mvarDecl.lctx.contains fvarId then true
-      else match lctx.find? fvarId with
-        | some (LocalDecl.ldecl ..) => false -- need expensive CheckAssignment.check
-        | _ =>
-          if fvars.any fun x => x.fvarId! == fvarId then true
-          else false -- We could throw an exception here, but we would have to use ExceptM. So, we let CheckAssignment.check do it
+      if mvarDecl.lctx.contains fvarId then return true
+      if let some (LocalDecl.ldecl ..) := lctx.find? fvarId then
+        return false -- need expensive CheckAssignment.check
+      if fvars.any fun x => x.fvarId! == fvarId then
+        return true
+      return false -- We could throw an exception here, but we would have to use ExceptM. So, we let CheckAssignment.check do it
     | .mvar mvarId'    =>
-      match mctx.getExprAssignmentCore? mvarId' with
-      | some _ => false -- use CheckAssignment.check to instantiate
-      | none   =>
-        if mvarId' == mvarId then false -- occurs check failed, use CheckAssignment.check to throw exception
-        else match mctx.findDecl? mvarId' with
-          | none           => false
-          | some mvarDecl' =>
-            if hasCtxLocals then false -- use CheckAssignment.check
-            else if mvarDecl'.lctx.isSubPrefixOf mvarDecl.lctx fvars then true
-            else false -- use CheckAssignment.check
+      let none := mctx.getExprAssignmentCore? mvarId' | return false -- use CheckAssignment.check to instantiate
+      if mvarId' == mvarId then return false -- occurs check failed, use CheckAssignment.check to throw exception
+      let some mvarDecl' := mctx.findDecl? mvarId' | return false
+      if hasCtxLocals then return false -- use CheckAssignment.check
+      if !mvarDecl'.lctx.isSubPrefixOf mvarDecl.lctx fvars then return false -- use CheckAssignment.check
+      let none := mctx.getDelayedMVarAssignmentCore? mvarId' | return false -- use CheckAssignment.check
+      return true
   visit e
 
 end CheckAssignmentQuick

src/Lean/Meta/LitValues.lean

@@ -99,6 +99,8 @@ def getUInt64Value? (e : Expr) : MetaM (Option UInt64) := OptionT.run do
   let (n, _) ← getOfNatValue? e ``UInt64
   return UInt64.ofNat n
 
+-- TODO: extensibility
+
 /--
 If `e` is a literal value, ensure it is encoded using the standard representation.
 Otherwise, just return `e`.
@@ -117,6 +119,23 @@ def normLitValue (e : Expr) : MetaM Expr := do
   if let some n ← getUInt64Value? e then return toExpr n
   return e
 
+/--
+Returns `true` if `e` is a literal value.
+-/
+def isLitValue (e : Expr) : MetaM Bool := do
+  let e ← instantiateMVars e
+  if (← getNatValue? e).isSome then return true
+  if (← getIntValue? e).isSome then return true
+  if (← getFinValue? e).isSome then return true
+  if (← getBitVecValue? e).isSome then return true
+  if (getStringValue? e).isSome then return true
+  if (← getCharValue? e).isSome then return true
+  if (← getUInt8Value? e).isSome then return true
+  if (← getUInt16Value? e).isSome then return true
+  if (← getUInt32Value? e).isSome then return true
+  if (← getUInt64Value? e).isSome then return true
+  return false
+
 /--
 If `e` is a `Nat`, `Int`, or `Fin` literal value, converts it into a constructor application.
 Otherwise, just return `e`.

src/Lean/Meta/SynthInstance.lean

@@ -561,11 +561,17 @@ def generate : SynthM Unit := do
     if backward.synthInstance.canonInstances.get (← getOptions) then
       unless gNode.typeHasMVars do
         if let some entry := (← get).tableEntries.find? key then
-          unless entry.answers.isEmpty do
+          if entry.answers.any fun answer => answer.result.numMVars == 0 then
             /-
-            We already have an answer for this node, and since its type does not have metavariables,
-            we can skip other solutions because we assume instances are "morally canonical".
+            We already have an answer that:
+              1. its result does not have metavariables.
+              2. its types do not have metavariables.
+
+            Thus, we can skip other solutions because we assume instances are "morally canonical".
             We have added this optimization to address issue #3996.
+
+            Remark: Condition 1 is important since root nodes only take into account results
+            that do **not** contain metavariables. This extra check was added to address issue #4213.
             -/
             modify fun s => { s with generatorStack := s.generatorStack.pop }
             return

src/Lean/Meta/Tactic/Apply.lean

@@ -8,6 +8,7 @@ import Lean.Util.FindMVar
 import Lean.Meta.SynthInstance
 import Lean.Meta.CollectMVars
 import Lean.Meta.Tactic.Util
+import Lean.PrettyPrinter
 
 namespace Lean.Meta
 /-- Controls which new mvars are turned in to goals by the `apply` tactic.
@@ -50,8 +51,15 @@ def getExpectedNumArgs (e : Expr) : MetaM Nat := do
   let (numArgs, _) ← getExpectedNumArgsAux e
   pure numArgs
 
-private def throwApplyError {α} (mvarId : MVarId) (eType : Expr) (targetType : Expr) : MetaM α :=
-  throwTacticEx `apply mvarId m!"failed to unify{indentExpr eType}\nwith{indentExpr targetType}"
+private def throwApplyError {α} (mvarId : MVarId) (eType : Expr) (targetType : Expr) : MetaM α := do
+  let explanation := MessageData.lazy
+    (f := fun ppctxt => ppctxt.runMetaM do
+        let (eType, targetType) ← addPPExplicitToExposeDiff eType targetType
+        return m!"{indentExpr eType}\nwith{indentExpr targetType}")
+    (hasSyntheticSorry := fun mvarctxt =>
+      (instantiateMVarsCore mvarctxt eType |>.1.hasSyntheticSorry) ||
+      (instantiateMVarsCore mvarctxt targetType |>.1.hasSyntheticSorry))
+  throwTacticEx `apply mvarId m!"failed to unify{explanation}"
 
 def synthAppInstances (tacticName : Name) (mvarId : MVarId) (newMVars : Array Expr) (binderInfos : Array BinderInfo)
     (synthAssignedInstances : Bool) (allowSynthFailures : Bool) : MetaM Unit :=

src/Lean/Meta/Tactic/Cases.lean

@@ -200,7 +200,7 @@ private def toCasesSubgoals (s : Array InductionSubgoal) (ctorNames : Array Name
     { ctorName           := ctorName,
       toInductionSubgoal := s }
 
-partial def unifyEqs? (numEqs : Nat) (mvarId : MVarId) (subst : FVarSubst) (caseName? : Option Name := none): MetaM (Option (MVarId × FVarSubst)) := do
+partial def unifyEqs? (numEqs : Nat) (mvarId : MVarId) (subst : FVarSubst) (caseName? : Option Name := none): MetaM (Option (MVarId × FVarSubst)) := withIncRecDepth do
   if numEqs == 0 then
     return some (mvarId, subst)
   else

src/Lean/Meta/Tactic/Grind.lean

@@ -6,6 +6,9 @@ Authors: Leonardo de Moura
 prelude
 import Lean.Meta.Tactic.Grind.Attr
 import Lean.Meta.Tactic.Grind.RevertAll
-import Lean.Meta.Tactic.Grind.EnsureNoMVar
 import Lean.Meta.Tactic.Grind.Types
 import Lean.Meta.Tactic.Grind.Preprocessor
+import Lean.Meta.Tactic.Grind.Util
+import Lean.Meta.Tactic.Grind.Cases
+import Lean.Meta.Tactic.Grind.Injection
+import Lean.Meta.Tactic.Grind.Core

src/Lean/Meta/Tactic/Grind/Attr.lean

@@ -16,4 +16,54 @@ builtin_initialize grindNormExt : SimpExtension ←
 builtin_initialize grindNormSimprocExt : SimprocExtension ←
   registerSimprocAttr `grind_norm_proc "simplification/normalization procedured for `grind`" none
 
+builtin_initialize grindCasesExt : SimpleScopedEnvExtension Name NameSet ←
+  registerSimpleScopedEnvExtension {
+    initial        := {}
+    addEntry       := fun s declName => s.insert declName
+  }
+
+/--
+Returns `true` if `declName` has been tagged with attribute `[grind_cases]`.
+-/
+def isGrindCasesTarget (declName : Name) : CoreM Bool :=
+  return grindCasesExt.getState (← getEnv) |>.contains declName
+
+private def getAlias? (value : Expr) : MetaM (Option Name) :=
+  lambdaTelescope value fun _ body => do
+    if let .const declName _ := body.getAppFn' then
+      return some declName
+    else
+      return none
+
+/--
+Throws an error if `declName` cannot be annotated with attribute `[grind_cases]`.
+We support the following cases:
+- `declName` is a non-recursive datatype.
+- `declName` is an abbreviation for a non-recursive datatype.
+-/
+private partial def validateGrindCasesAttr (declName : Name) : CoreM Unit := do
+  match (← getConstInfo declName) with
+  | .inductInfo info =>
+    if info.isRec then
+      throwError "`{declName}` is a recursive datatype"
+  | .defnInfo info =>
+    let failed := throwError "`{declName}` is a definition, but it is not an alias/abbreviation for an inductive datatype"
+    let some declName ← getAlias? info.value |>.run' {} {}
+      | failed
+    try
+      validateGrindCasesAttr declName
+    catch _ =>
+      failed
+  | _ =>
+    throwError "`{declName}` is not an inductive datatype or an alias for one"
+
+builtin_initialize
+  registerBuiltinAttribute {
+    name  := `grind_cases
+    descr := "`grind` tactic applies `cases` to (non-recursive) inductives during pre-processing step"
+    add   := fun declName _ attrKind => do
+      validateGrindCasesAttr declName
+      grindCasesExt.add declName attrKind
+  }
+
 end Lean.Meta.Grind

src/Lean/Meta/Tactic/Grind/Cases.lean

@@ -0,0 +1,62 @@
+/-
+Copyright (c) 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+Released under Apache 2.0 license as described in the file LICENSE.
+Authors: Leonardo de Moura
+-/
+prelude
+import Lean.Meta.Tactic.Cases
+
+namespace Lean.Meta.Grind
+/--
+The `grind` tactic includes an auxiliary `cases` tactic that is not intended for direct use by users.
+This method implements it.
+This tactic is automatically applied when introducing local declarations with a type tagged with `[grind_cases]`.
+It differs from the user-facing Lean `cases` tactic in the following ways:
+
+- It avoids unnecessary `revert` and `intro` operations.
+
+- It does not introduce new local declarations for each minor premise. Instead, the `grind` tactic preprocessor is responsible for introducing them.
+
+- It assumes that the major premise (i.e., the parameter `fvarId`) is the latest local declaration in the current goal.
+
+- If the major premise type is an indexed family, auxiliary declarations and (heterogeneous) equalities are introduced.
+  However, these equalities are not resolved using `unifyEqs`. Instead, the `grind` tactic employs union-find and
+  congruence closure to process these auxiliary equalities. This approach avoids applying substitution to propositions
+  that have already been internalized by `grind`.
+-/
+def cases (mvarId : MVarId) (fvarId : FVarId) : MetaM (List MVarId) := mvarId.withContext do
+  let tag ← mvarId.getTag
+  let type ← whnf (← fvarId.getType)
+  let .const declName _ := type.getAppFn | throwInductiveExpected type
+  let .inductInfo _ ← getConstInfo declName | throwInductiveExpected type
+  let recursorInfo ← mkRecursorInfo (mkCasesOnName declName)
+  let k (mvarId : MVarId) (fvarId : FVarId) (indices : Array Expr) (clearMajor : Bool) : MetaM (List MVarId) := do
+    let recursor ← mkRecursorAppPrefix mvarId `grind.cases fvarId recursorInfo indices
+    let mut recursor := mkApp (mkAppN recursor indices) (mkFVar fvarId)
+    let mut recursorType ← inferType recursor
+    let mut mvarIdsNew := #[]
+    for _ in [:recursorInfo.numMinors] do
+      let .forallE _ targetNew recursorTypeNew _ ← whnf recursorType
+        | throwTacticEx `grind.cases mvarId "unexpected recursor type"
+      recursorType := recursorTypeNew
+      let mvar ← mkFreshExprSyntheticOpaqueMVar targetNew tag
+      recursor := mkApp recursor mvar
+      let mvarIdNew ← if clearMajor then
+        mvar.mvarId!.clear fvarId
+      else
+        pure mvar.mvarId!
+      mvarIdsNew := mvarIdsNew.push mvarIdNew
+    mvarId.assign recursor
+    return mvarIdsNew.toList
+  if recursorInfo.numIndices > 0 then
+    let s ← generalizeIndices mvarId fvarId
+    s.mvarId.withContext do
+      k s.mvarId s.fvarId (s.indicesFVarIds.map mkFVar) (clearMajor := false)
+  else
+    let indices ← getMajorTypeIndices mvarId `grind.cases recursorInfo type
+    k mvarId fvarId indices (clearMajor := true)
+where
+  throwInductiveExpected {α} (type : Expr) : MetaM α := do
+    throwTacticEx `grind.cases mvarId m!"(non-recursive) inductive type expected at {mkFVar fvarId}{indentExpr type}"
+
+end Lean.Meta.Grind

src/Lean/Meta/Tactic/Grind/Core.lean

@@ -0,0 +1,157 @@
+/-
+Copyright (c) 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+Released under Apache 2.0 license as described in the file LICENSE.
+Authors: Leonardo de Moura
+-/
+prelude
+import Lean.Meta.Tactic.Grind.Types
+import Lean.Meta.LitValues
+
+namespace Lean.Meta.Grind
+/--
+Returns `true` if `e` is `True`, `False`, or a literal value.
+See `LitValues` for supported literals.
+-/
+def isInterpreted (e : Expr) : MetaM Bool := do
+  if e.isTrue || e.isFalse then return true
+  isLitValue e
+
+/--
+Creates an `ENode` for `e` if one does not already exist.
+This method assumes `e` has been hashconsed.
+-/
+def mkENode (e : Expr) (generation : Nat := 0) : GoalM Unit := do
+  if (← getENode? e).isSome then return ()
+  let ctor := (← isConstructorAppCore? e).isSome
+  let interpreted ← isInterpreted e
+  mkENodeCore e interpreted ctor generation
+
+/--
+Returns the root element in the equivalence class of `e`.
+-/
+def getRoot (e : Expr) : GoalM Expr := do
+  let some n ← getENode? e | return e
+  return n.root
+
+/--
+Returns the next element in the equivalence class of `e`.
+-/
+def getNext (e : Expr) : GoalM Expr := do
+  let some n ← getENode? e | return e
+  return n.next
+
+@[inline] def isSameExpr (a b : Expr) : Bool :=
+  -- It is safe to use pointer equality because we hashcons all expressions
+  -- inserted into the E-graph
+  unsafe ptrEq a b
+
+private def pushNewEqCore (lhs rhs proof : Expr) (isHEq : Bool) : GoalM Unit :=
+  modify fun s => { s with newEqs := s.newEqs.push { lhs, rhs, proof, isHEq } }
+
+@[inline] private def pushNewEq (lhs rhs proof : Expr) : GoalM Unit :=
+  pushNewEqCore lhs rhs proof (isHEq := false)
+
+@[inline] private def pushNewHEq (lhs rhs proof : Expr) : GoalM Unit :=
+  pushNewEqCore lhs rhs proof (isHEq := true)
+
+/--
+The fields `target?` and `proof?` in `e`'s `ENode` are encoding a transitivity proof
+from `e` to the root of the equivalence class
+This method "inverts" the proof, and makes it to go from the root of the equivalence class to `e`.
+
+We use this method when merging two equivalence classes.
+-/
+private partial def invertTrans (e : Expr) : GoalM Unit := do
+  go e false none none
+where
+  go (e : Expr) (flippedNew : Bool) (targetNew? : Option Expr) (proofNew? : Option Expr) : GoalM Unit := do
+    let some node ← getENode? e | unreachable!
+    if let some target := node.target? then
+      go target (!node.flipped) (some e) node.proof?
+    setENode e { node with
+      target? := targetNew?
+      flipped := flippedNew
+      proof?  := proofNew?
+    }
+
+private def markAsInconsistent : GoalM Unit :=
+  modify fun s => { s with inconsistent := true }
+
+private partial def addEqStep (lhs rhs proof : Expr) (isHEq : Bool) : GoalM Unit := do
+  let some lhsNode ← getENode? lhs | return () -- `lhs` has not been internalized yet
+  let some rhsNode ← getENode? rhs | return () -- `rhs` has not been internalized yet
+  if isSameExpr lhsNode.root rhsNode.root then return () -- `lhs` and `rhs` are already in the same equivalence class.
+  let some lhsRoot ← getENode? lhsNode.root | unreachable!
+  let some rhsRoot ← getENode? rhsNode.root | unreachable!
+  if    (lhsRoot.interpreted && !rhsRoot.interpreted)
+     || (lhsRoot.ctor && !rhsRoot.ctor)
+     || (lhsRoot.size > rhsRoot.size && !rhsRoot.interpreted && !rhsRoot.ctor) then
+    go rhs lhs rhsNode lhsNode rhsRoot lhsRoot true
+  else
+    go lhs rhs lhsNode rhsNode lhsRoot rhsRoot false
+where
+  go (lhs rhs : Expr) (lhsNode rhsNode lhsRoot rhsRoot : ENode) (flipped : Bool) : GoalM Unit := do
+    let mut valueInconsistency := false
+    if lhsRoot.interpreted && rhsRoot.interpreted then
+      if lhsNode.root.isTrue || rhsNode.root.isTrue then
+        markAsInconsistent
+      else
+        valueInconsistency := true
+    -- TODO: process valueInconsistency := true
+    /-
+    We have the following `target?/proof?`
+    `lhs -> ... -> lhsNode.root`
+    `rhs -> ... -> rhsNode.root`
+    We want to convert it to
+    `lhsNode.root -> ... -> lhs -*-> rhs -> ... -> rhsNode.root`
+    where step `-*->` is justified by `proof` (or `proof.symm` if `flipped := true`)
+    -/
+    invertTrans lhs
+    setENode lhs { lhsNode with
+      target? := rhs
+      proof?  := proof
+      flipped
+    }
+    -- TODO: Remove parents from congruence table
+    -- TODO: set propagateBool
+    updateRoots lhs rhsNode.root true -- TODO
+    -- TODO: Reinsert parents into congruence table
+    setENode lhsNode.root { lhsRoot with
+      next := rhsRoot.next
+    }
+    setENode rhsNode.root { rhsRoot with
+      next := lhsRoot.next
+      size := rhsRoot.size + lhsRoot.size
+      hasLambdas := rhsRoot.hasLambdas || lhsRoot.hasLambdas
+      heqProofs  := isHEq || rhsRoot.heqProofs || lhsRoot.heqProofs
+    }
+    -- TODO: copy parentst from lhsRoot parents to rhsRoot parents
+
+  updateRoots (lhs : Expr) (rootNew : Expr) (_propagateBool : Bool) : GoalM Unit := do
+    let rec loop (e : Expr) : GoalM Unit := do
+      -- TODO: propagateBool
+      let some n ← getENode? e | unreachable!
+      setENode e { n with root := rootNew }
+      if isSameExpr lhs n.next then return ()
+      loop n.next
+    loop lhs
+
+partial def addEqCore (lhs rhs proof : Expr) (isHEq : Bool) : GoalM Unit := do
+  addEqStep lhs rhs proof isHEq
+  processTodo
+where
+  processTodo : GoalM Unit := do
+    if (← get).inconsistent then
+      modify fun s => { s with newEqs := #[] }
+      return ()
+    let some { lhs, rhs, proof, isHEq } := (← get).newEqs.back? | return ()
+    addEqStep lhs rhs proof isHEq
+    processTodo
+
+def addEq (lhs rhs proof : Expr) : GoalM Unit := do
+  addEqCore lhs rhs proof false
+
+def addHEq (lhs rhs proof : Expr) : GoalM Unit := do
+  addEqCore lhs rhs proof true
+
+end Lean.Meta.Grind

src/Lean/Meta/Tactic/Grind/EnsureNoMVar.lean

@@ -1,19 +0,0 @@
-/-
-Copyright (c) 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-Released under Apache 2.0 license as described in the file LICENSE.
-Authors: Leonardo de Moura
--/
-prelude
-import Lean.Meta.Tactic.Util
-
-namespace Lean.Meta.Grind
-
-/--
-Throws an exception if target of the given goal contains metavariables.
--/
-def _root_.Lean.MVarId.ensureNoMVar (mvarId : MVarId) : MetaM Unit := do
-  let type ← instantiateMVars (← mvarId.getType)
-  if type.hasExprMVar then
-    throwTacticEx `grind mvarId "goal contains metavariables"
-
-end Lean.Meta.Grind

src/Lean/Meta/Tactic/Grind/Injection.lean

@@ -0,0 +1,39 @@
+/-
+Copyright (c) 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+Released under Apache 2.0 license as described in the file LICENSE.
+Authors: Leonardo de Moura
+-/
+prelude
+import Lean.Meta.CtorRecognizer
+import Lean.Meta.Tactic.Util
+import Lean.Meta.Tactic.Clear
+
+namespace Lean.Meta.Grind
+/--
+The `grind` tactic includes an auxiliary `injection?` tactic that is not intended for direct use by users.
+This tactic is automatically applied when introducing non-dependent propositions.
+It differs from the user-facing Lean `injection` tactic in the following ways:
+
+- It does not introduce new propositions. Instead, the `grind` tactic preprocessor is responsible for introducing them.
+- It assumes that `fvarId` is the latest local declaration in the current goal.
+- It does not handle cases where the constructors are different because the simplifier already reduces such equalities to `False`.
+- It does not have support for heterogeneous equality. Recall that the simplifier already reduces them to `Eq` if
+  the types are definitionally equal.
+-/
+def injection? (mvarId : MVarId) (fvarId : FVarId) : MetaM (Option MVarId) := mvarId.withContext do
+  let decl ← fvarId.getDecl
+  let_expr Eq _ a b := decl.type | return none
+  match (← isConstructorAppCore? a), (← isConstructorAppCore? b) with
+  | some aCtor, some bCtor =>
+    if aCtor.name != bCtor.name then return none -- Simplifier handles this case
+    let target ← mvarId.getType
+    let val ← mkNoConfusion target (mkFVar fvarId)
+    let .forallE _ targetNew _ _ ← whnfD (← inferType val) | return none
+    let targetNew := targetNew.headBeta
+    let tag ← mvarId.getTag
+    let mvarNew ← mkFreshExprSyntheticOpaqueMVar targetNew tag
+    mvarId.assign (mkApp val mvarNew)
+    return some (← mvarNew.mvarId!.clear fvarId)
+  | _, _ => return none
+
+end Lean.Meta.Grind

src/Lean/Meta/Tactic/Grind/Preprocessor.lean

@@ -4,14 +4,18 @@ Released under Apache 2.0 license as described in the file LICENSE.
 Authors: Leonardo de Moura
 -/
 prelude
+import Init.Grind.Lemmas
 import Lean.Meta.Canonicalizer
 import Lean.Meta.Tactic.Util
 import Lean.Meta.Tactic.Intro
 import Lean.Meta.Tactic.Simp.Main
 import Lean.Meta.Tactic.Grind.Attr
 import Lean.Meta.Tactic.Grind.RevertAll
-import Lean.Meta.Tactic.Grind.EnsureNoMVar
 import Lean.Meta.Tactic.Grind.Types
+import Lean.Meta.Tactic.Grind.Util
+import Lean.Meta.Tactic.Grind.Cases
+import Lean.Meta.Tactic.Grind.Injection
+import Lean.Meta.Tactic.Grind.Core
 
 namespace Lean.Meta.Grind
 namespace Preprocessor
@@ -25,99 +29,139 @@ structure Context where
   deriving Inhabited
 
 structure State where
-  canon : Canonicalizer.State := {}
-  todo  : List Goal := []
-  goals : PArray Goal := {}
   simpStats : Simp.Stats := {}
+  goals     : PArray Goal := {}
   deriving Inhabited
 
-structure Result where
-  canon : Canonicalizer.State := {}
-  goals : PArray Goal := {}
-  deriving Inhabited
+abbrev PreM := ReaderT Context $ StateRefT State GrindM
 
-abbrev M := ReaderT Context $ StateRefT State MetaM
-
-def mkInitialState (mvarId : MVarId) : State :=
-  { todo := [ mkGoal mvarId ] }
-
-def M.run (x : M α) (mvarId : MVarId) : MetaM α := do
+def PreM.run (x : PreM α) : GrindM α := do
   let thms ← grindNormExt.getTheorems
   let simprocs := #[(← grindNormSimprocExt.getSimprocs)]
   let simp : Simp.Context := {
+    config := { arith := true }
     simpTheorems := #[thms]
     congrTheorems := (← getSimpCongrTheorems)
   }
-  x { simp, simprocs } |>.run' (mkInitialState mvarId)
+  x { simp, simprocs } |>.run' {}
 
-def simpHyp? (mvarId : MVarId) (fvarId : FVarId) : M (Option (FVarId × MVarId)) := do
+def simp (_goal : Goal) (e : Expr) : PreM Simp.Result := do
+  -- TODO: use `goal` state in the simplifier
   let simpStats := (← get).simpStats
-  let (result, simpStats) ← simpLocalDecl mvarId fvarId (← read).simp (← read).simprocs (stats := simpStats)
+  let (r, simpStats) ← Meta.simp e (← read).simp (← read).simprocs (stats := simpStats)
   modify fun s => { s with simpStats }
-  return result
-
-def getNextGoal? : M (Option Goal) := do
-  match (← get).todo with
-  | [] => return none
-  | goal :: todo =>
-    modify fun s => { s with todo }
-    return some goal
+  return r
 
 inductive IntroResult where
-  | done | closed
+  | done
   | newHyp (fvarId : FVarId) (goal : Goal)
+  | newDepHyp (goal : Goal)
   | newLocal (fvarId : FVarId) (goal : Goal)
 
-def introNext (goal : Goal) : M IntroResult := do
+def introNext (goal : Goal) : PreM IntroResult := do
   let target ← goal.mvarId.getType
   if target.isArrow then
-    let (fvarId, mvarId) ← goal.mvarId.intro1P
-    -- TODO: canonicalize subterms
-    mvarId.withContext do
-    if (← isProp (← fvarId.getType)) then
-      let some (fvarId, mvarId) ← simpHyp? mvarId fvarId | return .closed
-      return .newHyp fvarId { goal with mvarId }
-    else
-      return .newLocal fvarId { goal with mvarId }
+    goal.mvarId.withContext do
+      let p := target.bindingDomain!
+      if !(← isProp p) then
+        let (fvarId, mvarId) ← goal.mvarId.intro1P
+        return .newLocal fvarId { goal with mvarId }
+      else
+        let tag ← goal.mvarId.getTag
+        let q := target.bindingBody!
+        let r ← simp goal p
+        let p' := r.expr
+        let p' ← canon p'
+        let p' ← shareCommon p'
+        let fvarId ← mkFreshFVarId
+        let lctx := (← getLCtx).mkLocalDecl fvarId target.bindingName! p' target.bindingInfo!
+        let mvarNew ← mkFreshExprMVarAt lctx (← getLocalInstances) q .syntheticOpaque tag
+        let mvarIdNew := mvarNew.mvarId!
+        mvarIdNew.withContext do
+          let h ← mkLambdaFVars #[mkFVar fvarId] mvarNew
+          match r.proof? with
+          | some he =>
+            let hNew := mkAppN (mkConst ``Lean.Grind.intro_with_eq) #[p, p', q, he, h]
+            goal.mvarId.assign hNew
+            return .newHyp fvarId { goal with mvarId := mvarIdNew }
+          | none =>
+            -- `p` and `p'` are definitionally equal
+            goal.mvarId.assign h
+            return .newHyp fvarId { goal with mvarId := mvarIdNew }
   else if target.isLet || target.isForall then
-    -- TODO: canonicalize subterms
-    -- TODO: If forall is of the form `∀ h : <proposition>, A h`, generalize `h`.
     let (fvarId, mvarId) ← goal.mvarId.intro1P
-    return .newLocal fvarId { goal with mvarId }
+    mvarId.withContext do
+      let localDecl ← fvarId.getDecl
+      if (← isProp localDecl.type) then
+        -- Add a non-dependent copy
+        let mvarId ← mvarId.assert localDecl.userName localDecl.type (mkFVar fvarId)
+        return .newDepHyp { goal with mvarId }
+      else
+        return .newLocal fvarId { goal with mvarId }
   else
     return .done
 
-def pushTodo (goal : Goal) : M Unit :=
-  modify fun s => { s with todo := goal :: s.todo }
-
-def pushResult (goal : Goal) : M Unit :=
+def pushResult (goal : Goal) : PreM Unit :=
   modify fun s => { s with goals := s.goals.push goal }
 
-partial def main (mvarId : MVarId) : MetaM Result := do
-  mvarId.ensureNoMVar
-  let mvarId ← mvarId.revertAll
-  mvarId.ensureNoMVar
-  let s ← (loop *> get) |>.run mvarId
-  return { s with }
-where
-  loop : M Unit := do
-    let some goal ← getNextGoal? | return ()
-    trace[Meta.debug] "{goal.mvarId}"
-    match (← introNext goal) with
-    | .closed => loop
-    | .done =>
-      -- TODO: apply `byContradiction`
+def isCasesCandidate (fvarId : FVarId) : MetaM Bool := do
+  let .const declName _ := (← fvarId.getType).getAppFn | return false
+  isGrindCasesTarget declName
+
+def applyCases? (goal : Goal) (fvarId : FVarId) : MetaM (Option (List Goal)) := goal.mvarId.withContext do
+  if (← isCasesCandidate fvarId) then
+    let mvarIds ← cases goal.mvarId fvarId
+    return mvarIds.map fun mvarId => { goal with mvarId }
+  else
+    return none
+
+def applyInjection? (goal : Goal) (fvarId : FVarId) : MetaM (Option Goal) := do
+  if let some mvarId ← injection? goal.mvarId fvarId then
+    return some { goal with mvarId }
+  else
+    return none
+
+partial def loop (goal : Goal) : PreM Unit := do
+  match (← introNext goal) with
+  | .done =>
+    if let some mvarId ← goal.mvarId.byContra? then
+      loop { goal with mvarId }
+    else
       pushResult goal
-      return ()
-    | .newHyp fvarId goal =>
-      -- TODO: apply eliminators
+  | .newHyp fvarId goal =>
+    if let some goals ← applyCases? goal fvarId then
+      goals.forM loop
+    else if let some goal ← applyInjection? goal fvarId then
+      loop goal
+    else
       let clause ← goal.mvarId.withContext do mkInputClause fvarId
-      pushTodo { goal with clauses := goal.clauses.push clause }
-      loop
-    | .newLocal _ goal =>
-      -- TODO: apply eliminators
-      pushTodo goal
-      loop
+      loop { goal with clauses := goal.clauses.push clause }
+  | .newDepHyp goal =>
+    loop goal
+  | .newLocal fvarId goal =>
+    if let some goals ← applyCases? goal fvarId then
+      goals.forM loop
+    else
+      loop goal
+
+def preprocess (mvarId : MVarId) : PreM State := do
+  loop (← mkGoal mvarId)
+  get
 
 end Preprocessor
+
+open Preprocessor
+
+partial def main (mvarId : MVarId) (mainDeclName : Name) : MetaM (List MVarId) := do
+  mvarId.ensureProp
+  mvarId.ensureNoMVar
+  let mvarId ← mvarId.clearAuxDecls
+  let mvarId ← mvarId.revertAll
+  mvarId.ensureNoMVar
+  let mvarId ← mvarId.abstractNestedProofs mainDeclName
+  let mvarId ← mvarId.unfoldReducible
+  let mvarId ← mvarId.betaReduce
+  let s ← preprocess mvarId |>.run |>.run mainDeclName
+  return s.goals.toList.map (·.mvarId)
+
 end Lean.Meta.Grind

src/Lean/Meta/Tactic/Grind/Types.lean

@@ -4,11 +4,91 @@ Released under Apache 2.0 license as described in the file LICENSE.
 Authors: Leonardo de Moura
 -/
 prelude
+import Lean.Util.ShareCommon
 import Lean.Meta.Basic
+import Lean.Meta.AbstractNestedProofs
+import Lean.Meta.Canonicalizer
 import Lean.Meta.Tactic.Util
 
 namespace Lean.Meta.Grind
 
+structure Context where
+  mainDeclName : Name
+
+structure State where
+  canon      : Canonicalizer.State := {}
+  /-- `ShareCommon` (aka `Hashconsing`) state. -/
+  scState    : ShareCommon.State.{0} ShareCommon.objectFactory := ShareCommon.State.mk _
+  /-- Next index for creating auxiliary theorems. -/
+  nextThmIdx : Nat := 1
+
+abbrev GrindM := ReaderT Context $ StateRefT State MetaM
+
+@[inline] def GrindM.run (x : GrindM α) (mainDeclName : Name) : MetaM α :=
+  x { mainDeclName } |>.run' {}
+
+def abstractNestedProofs (e : Expr) : GrindM Expr := do
+  let nextIdx := (← get).nextThmIdx
+  let (e, s') ← AbstractNestedProofs.visit e |>.run { baseName := (← read).mainDeclName } |>.run |>.run { nextIdx }
+  modify fun s => { s with nextThmIdx := s'.nextIdx }
+  return e
+
+def shareCommon (e : Expr) : GrindM Expr := do
+  modifyGet fun { canon, scState, nextThmIdx } =>
+    let (e, scState) := ShareCommon.State.shareCommon scState e
+    (e, { canon, scState, nextThmIdx })
+
+def canon (e : Expr) : GrindM Expr := do
+  let canonS ← modifyGet fun s => (s.canon, { s with canon := {} })
+  let (e, canonS) ← Canonicalizer.CanonM.run (canonRec e) (s := canonS)
+  modify fun s => { s with canon := canonS }
+  return e
+where
+  canonRec (e : Expr) : CanonM Expr := do
+    let post (e : Expr) : CanonM TransformStep := do
+      if e.isApp then
+        return .done (← Meta.canon e)
+      else
+        return .done e
+    transform e post
+
+/--
+Stores information for a node in the egraph.
+Each internalized expression `e` has an `ENode` associated with it.
+-/
+structure ENode where
+  /-- Next element in the equivalence class. -/
+  next : Expr
+  /-- Root (aka canonical representative) of the equivalence class -/
+  root : Expr
+  /-- Root of the congruence class. This is field is a don't care if `e` is not an application. -/
+  cgRoot : Expr
+  /--
+  When `e` was added to this equivalence class because of an equality `h : e = target`,
+  then we store `target` here, and `h` at `proof?`.
+  -/
+  target? : Option Expr := none
+  proof? : Option Expr := none
+  /-- Proof has been flipped. -/
+  flipped : Bool := false
+  /-- Number of elements in the equivalence class, this field is meaningless if node is not the root. -/
+  size : Nat := 1
+  /-- `interpreted := true` if node should be viewed as an abstract value. -/
+  interpreted : Bool := false
+  /-- `ctor := true` if the head symbol is a constructor application. -/
+  ctor : Bool := false
+  /-- `hasLambdas := true` if equivalence class contains lambda expressions. -/
+  hasLambdas : Bool := false
+  /--
+  If `heqProofs := true`, then some proofs in the equivalence class are based
+  on heterogeneous equality.
+  -/
+  heqProofs : Bool := false
+  generation : Nat := 0
+  /-- Modification time -/
+  mt : Nat := 0
+  -- TODO: see Lean 3 implementation
+
 structure Clause where
   expr  : Expr
   proof : Expr
@@ -17,15 +97,57 @@ structure Clause where
 def mkInputClause (fvarId : FVarId) : MetaM Clause :=
   return { expr := (← fvarId.getType), proof := mkFVar fvarId }
 
-structure Goal where
-  mvarId  : MVarId
-  clauses : PArray Clause := {}
-  deriving Inhabited
+structure NewEq where
+  lhs   : Expr
+  rhs   : Expr
+  proof : Expr
+  isHEq : Bool
 
-def mkGoal (mvarId : MVarId) : Goal :=
-  { mvarId }
+structure Goal where
+  mvarId       : MVarId
+  clauses      : PArray Clause := {}
+  enodes       : PHashMap USize ENode := {}
+  newEqs       : Array NewEq := #[]
+  /-- `inconsistent := true` if `ENode`s for `True` and `False` are in the same equivalence class. -/
+  inconsistent : Bool := false
+  /-- Goal modification time. -/
+  gmt          : Nat := 0
+  deriving Inhabited
 
 def Goal.admit (goal : Goal) : MetaM Unit :=
   goal.mvarId.admit
 
+abbrev GoalM := StateRefT Goal GrindM
+
+@[inline] def GoalM.run (goal : Goal) (x : GoalM α) : GrindM (α × Goal) :=
+  StateRefT'.run x goal
+
+@[inline] def GoalM.run' (goal : Goal) (x : GoalM Unit) : GrindM Goal :=
+  StateRefT'.run' (x *> get) goal
+
+/--
+Returns `some n` if `e` has already been "internalized" into the
+Otherwise, returns `none`s.
+-/
+def getENode? (e : Expr) : GoalM (Option ENode) :=
+  return (← get).enodes.find? (unsafe ptrAddrUnsafe e)
+
+def setENode (e : Expr) (n : ENode) : GoalM Unit :=
+  modify fun s => { s with enodes := s.enodes.insert (unsafe ptrAddrUnsafe e) n }
+
+def mkENodeCore (e : Expr) (interpreted ctor : Bool) (generation : Nat) : GoalM Unit := do
+  setENode e {
+    next := e, root := e, cgRoot := e, size := 1
+    flipped := false
+    heqProofs := false
+    hasLambdas := e.isLambda
+    mt := (← get).gmt
+    interpreted, ctor, generation
+  }
+
+def mkGoal (mvarId : MVarId) : GrindM Goal := do
+  GoalM.run' { mvarId } do
+    mkENodeCore (← shareCommon (mkConst ``True)) (interpreted := true) (ctor := false) (generation := 0)
+    mkENodeCore (← shareCommon (mkConst ``False)) (interpreted := true) (ctor := false) (generation := 0)
+
 end Lean.Meta.Grind

src/Lean/Meta/Tactic/Grind/Util.lean

@@ -0,0 +1,99 @@
+/-
+Copyright (c) 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+Released under Apache 2.0 license as described in the file LICENSE.
+Authors: Leonardo de Moura
+-/
+prelude
+import Lean.Meta.AbstractNestedProofs
+import Lean.Meta.Tactic.Util
+import Lean.Meta.Tactic.Clear
+
+namespace Lean.Meta.Grind
+/--
+Throws an exception if target of the given goal contains metavariables.
+-/
+def _root_.Lean.MVarId.ensureNoMVar (mvarId : MVarId) : MetaM Unit := do
+  let type ← instantiateMVars (← mvarId.getType)
+  if type.hasExprMVar then
+    throwTacticEx `grind mvarId "goal contains metavariables"
+
+/--
+Throws an exception if target is not a proposition.
+-/
+def _root_.Lean.MVarId.ensureProp (mvarId : MVarId) : MetaM Unit := do
+  let type ← mvarId.getType
+  unless (← isProp type) do
+    throwTacticEx `grind mvarId "goal is not a proposition"
+
+def _root_.Lean.MVarId.transformTarget (mvarId : MVarId) (f : Expr → MetaM Expr) : MetaM MVarId := mvarId.withContext do
+  mvarId.checkNotAssigned `grind
+  let tag ← mvarId.getTag
+  let type ← mvarId.getType
+  let typeNew ← f type
+  let mvarNew ← mkFreshExprSyntheticOpaqueMVar typeNew tag
+  mvarId.assign mvarNew
+  return mvarNew.mvarId!
+
+/--
+Unfold all `reducible` declarations occurring in `e`.
+-/
+def unfoldReducible (e : Expr) : MetaM Expr :=
+  let pre (e : Expr) : MetaM TransformStep := do
+    let .const declName _ := e.getAppFn | return .continue
+    unless (← isReducible declName) do return .continue
+    let some v ← unfoldDefinition? e | return .continue
+    return .visit v
+  Core.transform e (pre := pre)
+
+/--
+Unfold all `reducible` declarations occurring in the goal's target.
+-/
+def _root_.Lean.MVarId.unfoldReducible (mvarId : MVarId) : MetaM MVarId :=
+  mvarId.transformTarget Grind.unfoldReducible
+
+/--
+Abstract nested proofs occurring in the goal's target.
+-/
+def _root_.Lean.MVarId.abstractNestedProofs (mvarId : MVarId) (mainDeclName : Name) : MetaM MVarId :=
+  mvarId.transformTarget (Lean.Meta.abstractNestedProofs mainDeclName)
+
+/--
+Beta-reduce the goal's target.
+-/
+def _root_.Lean.MVarId.betaReduce (mvarId : MVarId) : MetaM MVarId :=
+  mvarId.transformTarget (Core.betaReduce ·)
+
+/--
+If the target is not `False`, apply `byContradiction`.
+-/
+def _root_.Lean.MVarId.byContra? (mvarId : MVarId) : MetaM (Option MVarId) := mvarId.withContext do
+  mvarId.checkNotAssigned `grind.by_contra
+  let target ← mvarId.getType
+  if target.isFalse then return none
+  let targetNew ← mkArrow (mkNot target) (mkConst ``False)
+  let tag ← mvarId.getTag
+  let mvarNew ← mkFreshExprSyntheticOpaqueMVar targetNew tag
+  mvarId.assign <| mkApp2 (mkConst ``Classical.byContradiction) target mvarNew
+  return mvarNew.mvarId!
+
+/--
+Clear auxiliary decls used to encode recursive declarations.
+`grind` eliminates them to ensure they are not accidentaly used by its proof automation.
+-/
+def _root_.Lean.MVarId.clearAuxDecls (mvarId : MVarId) : MetaM MVarId := mvarId.withContext do
+  mvarId.checkNotAssigned `grind.clear_aux_decls
+  let mut toClear := []
+  for localDecl in (← getLCtx) do
+    if localDecl.isAuxDecl then
+      toClear := localDecl.fvarId :: toClear
+  if toClear.isEmpty then
+    return mvarId
+  let mut mvarId := mvarId
+  for fvarId in toClear do
+    try
+      mvarId ← mvarId.clear fvarId
+    catch _ =>
+      throwTacticEx `grind.clear_aux_decls mvarId "failed to clear local auxiliary declaration"
+  return mvarId
+
+end Lean.Meta.Grind

src/Lean/Meta/Tactic/Induction.lean

@@ -120,8 +120,75 @@ private partial def finalize
       pure subgoals
   loop (recursorInfo.paramsPos.length + 1) 0 recursor recursorType false #[]
 
-private def throwUnexpectedMajorType {α} (mvarId : MVarId) (majorType : Expr) : MetaM α :=
-  throwTacticEx `induction mvarId m!"unexpected major premise type{indentExpr majorType}"
+private def throwUnexpectedMajorType (tacticName : Name) (mvarId : MVarId) (majorType : Expr) : MetaM α :=
+  throwTacticEx tacticName mvarId m!"unexpected major premise type{indentExpr majorType}"
+
+/--
+Auxiliary method for implementing `induction`-like tactics.
+It retrieves indices from `majorType` using `recursorInfo`.
+Remark: `mvarId` and `tacticName` are used to generate error messages.
+-/
+def getMajorTypeIndices (mvarId : MVarId) (tacticName : Name) (recursorInfo : RecursorInfo) (majorType : Expr) : MetaM (Array Expr) := do
+  let majorTypeArgs := majorType.getAppArgs
+  recursorInfo.indicesPos.toArray.mapM fun idxPos => do
+    if idxPos ≥ majorTypeArgs.size then throwTacticEx tacticName mvarId m!"major premise type is ill-formed{indentExpr majorType}"
+    let idx := majorTypeArgs.get! idxPos
+    unless idx.isFVar do throwTacticEx tacticName mvarId m!"major premise type index {idx} is not a variable{indentExpr majorType}"
+    majorTypeArgs.size.forM fun i => do
+      let arg := majorTypeArgs[i]!
+      if i != idxPos && arg == idx then
+        throwTacticEx tacticName mvarId m!"'{idx}' is an index in major premise, but it occurs more than once{indentExpr majorType}"
+      if i < idxPos then
+        if (← exprDependsOn arg idx.fvarId!) then
+          throwTacticEx tacticName mvarId m!"'{idx}' is an index in major premise, but it occurs in previous arguments{indentExpr majorType}"
+      -- If arg is also and index and a variable occurring after `idx`, we need to make sure it doesn't depend on `idx`.
+      -- Note that if `arg` is not a variable, we will fail anyway when we visit it.
+      if i > idxPos && recursorInfo.indicesPos.contains i && arg.isFVar then
+        let idxDecl ← idx.fvarId!.getDecl
+        if (← localDeclDependsOn idxDecl arg.fvarId!) then
+          throwTacticEx tacticName mvarId m!"'{idx}' is an index in major premise, but it depends on index occurring at position #{i+1}"
+    return idx
+
+/--
+Auxiliary method for implementing `induction`-like tactics.
+It creates the prefix of a recursor application up-to `motive`.
+The motive is computed by abstracting `major` and `indices` at `mvarId.getType`.
+It retrieves indices from `majorType` using `recursorInfo`.
+Remark: `mvarId` and `tacticName` are used to generate error messages.
+-/
+def mkRecursorAppPrefix (mvarId : MVarId) (tacticName : Name) (majorFVarId : FVarId) (recursorInfo : RecursorInfo) (indices : Array Expr) : MetaM Expr := do
+  let major := mkFVar majorFVarId
+  let target ← mvarId.getType
+  let targetLevel ← getLevel target
+  let targetLevel ← normalizeLevel targetLevel
+  let majorLocalDecl ← majorFVarId.getDecl
+  let some majorType ← whnfUntil majorLocalDecl.type recursorInfo.typeName
+    | throwUnexpectedMajorType tacticName mvarId majorLocalDecl.type
+  majorType.withApp fun majorTypeFn majorTypeArgs => do
+    match majorTypeFn with
+    | .const _ majorTypeFnLevels => do
+      let majorTypeFnLevels := majorTypeFnLevels.toArray
+      let (recursorLevels, foundTargetLevel) ← recursorInfo.univLevelPos.foldlM (init := (#[], false))
+          fun (recursorLevels, foundTargetLevel) (univPos : RecursorUnivLevelPos) => do
+            match univPos with
+            | RecursorUnivLevelPos.motive => pure (recursorLevels.push targetLevel, true)
+            | RecursorUnivLevelPos.majorType idx =>
+              if idx ≥ majorTypeFnLevels.size then throwTacticEx tacticName mvarId "ill-formed recursor"
+              pure (recursorLevels.push (majorTypeFnLevels.get! idx), foundTargetLevel)
+      if !foundTargetLevel && !targetLevel.isZero then
+        throwTacticEx tacticName mvarId m!"recursor '{recursorInfo.recursorName}' can only eliminate into Prop"
+      let recursor := mkConst recursorInfo.recursorName recursorLevels.toList
+      let recursor ← addRecParams mvarId majorTypeArgs recursorInfo.paramsPos recursor
+      -- Compute motive
+      let motive := target
+      let motive ← if recursorInfo.depElim then
+        pure <| mkLambda `x BinderInfo.default (← inferType major) (← motive.abstractM #[major])
+      else
+        pure motive
+      let motive ← mkLambdaFVars indices motive
+      return mkApp recursor motive
+    | _ =>
+      throwTacticEx tacticName mvarId "major premise is not of the form (C ...)"
 
 def _root_.Lean.MVarId.induction (mvarId : MVarId) (majorFVarId : FVarId) (recursorName : Name) (givenNames : Array AltVarNames := #[]) : MetaM (Array InductionSubgoal) :=
   mvarId.withContext do
@@ -129,30 +196,14 @@ def _root_.Lean.MVarId.induction (mvarId : MVarId) (majorFVarId : FVarId) (recur
     mvarId.checkNotAssigned `induction
     let majorLocalDecl ← majorFVarId.getDecl
     let recursorInfo ← mkRecursorInfo recursorName
-    let some majorType ← whnfUntil majorLocalDecl.type recursorInfo.typeName | throwUnexpectedMajorType mvarId majorLocalDecl.type
+    let some majorType ← whnfUntil majorLocalDecl.type recursorInfo.typeName
+      | throwUnexpectedMajorType `induction mvarId majorLocalDecl.type
     majorType.withApp fun _ majorTypeArgs => do
       recursorInfo.paramsPos.forM fun paramPos? => do
         match paramPos? with
         | none          => pure ()
         | some paramPos => if paramPos ≥ majorTypeArgs.size then throwTacticEx `induction mvarId m!"major premise type is ill-formed{indentExpr majorType}"
-      let indices ← recursorInfo.indicesPos.toArray.mapM fun idxPos => do
-        if idxPos ≥ majorTypeArgs.size then throwTacticEx `induction mvarId m!"major premise type is ill-formed{indentExpr majorType}"
-        let idx := majorTypeArgs.get! idxPos
-        unless idx.isFVar do throwTacticEx `induction mvarId m!"major premise type index {idx} is not a variable{indentExpr majorType}"
-        majorTypeArgs.size.forM fun i => do
-          let arg := majorTypeArgs[i]!
-          if i != idxPos && arg == idx then
-            throwTacticEx `induction mvarId m!"'{idx}' is an index in major premise, but it occurs more than once{indentExpr majorType}"
-          if i < idxPos then
-            if (← exprDependsOn arg idx.fvarId!) then
-              throwTacticEx `induction mvarId m!"'{idx}' is an index in major premise, but it occurs in previous arguments{indentExpr majorType}"
-          -- If arg is also and index and a variable occurring after `idx`, we need to make sure it doesn't depend on `idx`.
-          -- Note that if `arg` is not a variable, we will fail anyway when we visit it.
-          if i > idxPos && recursorInfo.indicesPos.contains i && arg.isFVar then
-            let idxDecl ← idx.fvarId!.getDecl
-            if (← localDeclDependsOn idxDecl arg.fvarId!) then
-              throwTacticEx `induction mvarId m!"'{idx}' is an index in major premise, but it depends on index occurring at position #{i+1}"
-        pure idx
+      let indices ← getMajorTypeIndices mvarId `induction recursorInfo majorType
       let target ← mvarId.getType
       if (← pure !recursorInfo.depElim <&&> exprDependsOn target majorFVarId) then
         throwTacticEx `induction mvarId m!"recursor '{recursorName}' does not support dependent elimination, but conclusion depends on major premise"
@@ -175,37 +226,8 @@ def _root_.Lean.MVarId.induction (mvarId : MVarId) (majorFVarId : FVarId) (recur
       let majorFVarId := majorFVarId'
       let major := mkFVar majorFVarId
       mvarId.withContext do
-        let target ← mvarId.getType
-        let targetLevel ← getLevel target
-        let targetLevel ← normalizeLevel targetLevel
-        let majorLocalDecl ← majorFVarId.getDecl
-        let some majorType ← whnfUntil majorLocalDecl.type recursorInfo.typeName | throwUnexpectedMajorType mvarId majorLocalDecl.type
-        majorType.withApp fun majorTypeFn majorTypeArgs => do
-          match majorTypeFn with
-          | Expr.const _ majorTypeFnLevels => do
-            let majorTypeFnLevels := majorTypeFnLevels.toArray
-            let (recursorLevels, foundTargetLevel) ← recursorInfo.univLevelPos.foldlM (init := (#[], false))
-                fun (recursorLevels, foundTargetLevel) (univPos : RecursorUnivLevelPos) => do
-                  match univPos with
-                  | RecursorUnivLevelPos.motive => pure (recursorLevels.push targetLevel, true)
-                  | RecursorUnivLevelPos.majorType idx =>
-                    if idx ≥ majorTypeFnLevels.size then throwTacticEx `induction mvarId "ill-formed recursor"
-                    pure (recursorLevels.push (majorTypeFnLevels.get! idx), foundTargetLevel)
-            if !foundTargetLevel && !targetLevel.isZero then
-              throwTacticEx `induction mvarId m!"recursor '{recursorName}' can only eliminate into Prop"
-            let recursor := mkConst recursorName recursorLevels.toList
-            let recursor ← addRecParams mvarId majorTypeArgs recursorInfo.paramsPos recursor
-            -- Compute motive
-            let motive := target
-            let motive ← if recursorInfo.depElim then
-              pure <| mkLambda `x BinderInfo.default (← inferType major) (← motive.abstractM #[major])
-            else
-              pure motive
-            let motive ← mkLambdaFVars indices motive
-            let recursor := mkApp recursor motive
-            finalize mvarId givenNames recursorInfo reverted major indices baseSubst recursor
-          | _ =>
-           throwTacticEx `induction mvarId "major premise is not of the form (C ...)"
+        let recursor ← mkRecursorAppPrefix mvarId `induction majorFVarId recursorInfo indices
+        finalize mvarId givenNames recursorInfo reverted major indices baseSubst recursor
 
 @[deprecated MVarId.induction (since := "2022-07-15")]
 def induction (mvarId : MVarId) (majorFVarId : FVarId) (recursorName : Name) (givenNames : Array AltVarNames := #[]) : MetaM (Array InductionSubgoal) :=

src/Lean/Meta/Tactic/Simp/BuiltinSimprocs/Char.lean

@@ -45,6 +45,11 @@ builtin_dsimproc [simp, seval] reduceVal (Char.val _) := fun e => do
   let_expr Char.val arg ← e | return .continue
   let some c ← fromExpr? arg | return .continue
   return .done <| toExpr c.val
+
+builtin_simproc [simp, seval] reduceLT  (( _ : Char) < _)  := reduceBinPred ``LT.lt 4 (. < .)
+builtin_simproc [simp, seval] reduceLE  (( _ : Char) ≤ _)  := reduceBinPred ``LE.le 4 (. ≤ .)
+builtin_simproc [simp, seval] reduceGT  (( _ : Char) > _)  := reduceBinPred ``GT.gt 4 (. > .)
+builtin_simproc [simp, seval] reduceGE  (( _ : Char) ≥ _)  := reduceBinPred ``GE.ge 4 (. ≥ .)
 builtin_simproc [simp, seval] reduceEq  (( _ : Char) = _)  := reduceBinPred ``Eq 3 (. = .)
 builtin_simproc [simp, seval] reduceNe  (( _ : Char) ≠ _)  := reduceBinPred ``Ne 3 (. ≠ .)
 builtin_dsimproc [simp, seval] reduceBEq  (( _ : Char) == _)  := reduceBoolPred ``BEq.beq 4 (. == .)

src/Lean/Meta/Tactic/Simp/BuiltinSimprocs/String.lean

@@ -32,4 +32,25 @@ builtin_dsimproc [simp, seval] reduceMk (String.mk _) := fun e => do
   unless e.isAppOfArity ``String.mk 1 do return .continue
   reduceListChar e.appArg! ""
 
+@[inline] def reduceBinPred (declName : Name) (arity : Nat) (op : String → String → Bool) (e : Expr) : SimpM Step := do
+  unless e.isAppOfArity declName arity do return .continue
+  let some n ← fromExpr? e.appFn!.appArg! | return .continue
+  let some m ← fromExpr? e.appArg! | return .continue
+  evalPropStep e (op n m)
+
+@[inline] def reduceBoolPred (declName : Name) (arity : Nat) (op : String → String → Bool) (e : Expr) : SimpM DStep := do
+  unless e.isAppOfArity declName arity do return .continue
+  let some n ← fromExpr? e.appFn!.appArg! | return .continue
+  let some m ← fromExpr? e.appArg! | return .continue
+  return .done <| toExpr (op n m)
+
+builtin_simproc [simp, seval] reduceLT  (( _ : String) < _)  := reduceBinPred ``LT.lt 4 (. < .)
+builtin_simproc [simp, seval] reduceLE  (( _ : String) ≤ _)  := reduceBinPred ``LE.le 4 (. ≤ .)
+builtin_simproc [simp, seval] reduceGT  (( _ : String) > _)  := reduceBinPred ``GT.gt 4 (. > .)
+builtin_simproc [simp, seval] reduceGE  (( _ : String) ≥ _)  := reduceBinPred ``GE.ge 4 (. ≥ .)
+builtin_simproc [simp, seval] reduceEq  (( _ : String) = _)  := reduceBinPred ``Eq 3 (. = .)
+builtin_simproc [simp, seval] reduceNe  (( _ : String) ≠ _)  := reduceBinPred ``Ne 3 (. ≠ .)
+builtin_dsimproc [simp, seval] reduceBEq  (( _ : String) == _)  := reduceBoolPred ``BEq.beq 4 (. == .)
+builtin_dsimproc [simp, seval] reduceBNe  (( _ : String) != _)  := reduceBoolPred ``bne 4 (. != .)
+
 end String

src/Lean/Meta/Tactic/UnifyEq.lean

@@ -21,6 +21,11 @@ structure UnifyEqResult where
   subst     : FVarSubst
   numNewEqs : Nat := 0
 
+private def toOffset? (e : Expr) : MetaM (Option (Expr × Nat)) := do
+  match (← evalNat e) with
+  | some k => return some (mkNatLit 0, k)
+  | none => isOffset? e
+
 /--
   Helper method for methods such as `Cases.unifyEqs?`.
   Given the given goal `mvarId` containing the local hypothesis `eqFVarId`, it performs the following operations:
@@ -69,7 +74,30 @@ def unifyEq? (mvarId : MVarId) (eqFVarId : FVarId) (subst : FVarSubst := {})
             return none -- this alternative has been solved
           else
             throwError "dependent elimination failed, failed to solve equation{indentExpr eqDecl.type}"
+        /- Special support for offset equalities -/
+        let injectionOffset? (a b : Expr) := do
+          unless (← getEnv).contains ``Nat.elimOffset do return none
+          let some (xa, ka) ← toOffset? a | return none
+          let some (xb, kb) ← toOffset? b | return none
+          if ka == 0 || kb == 0 then return none -- use default noConfusion
+          let (x, y, k) ← if ka < kb then
+            pure (xa, (← mkAdd xb (mkNatLit (kb - ka))), ka)
+          else if ka = kb then
+            pure (xa, xb, ka)
+          else
+            pure ((← mkAdd xa (mkNatLit (ka - kb))), xb, kb)
+          let target ← mvarId.getType
+          let u ← getLevel target
+          let newTarget ← mkArrow (← mkEq x y) target
+          let tag ← mvarId.getTag
+          let newMVar ← mkFreshExprSyntheticOpaqueMVar newTarget tag
+          let val := mkAppN (mkConst ``Nat.elimOffset [u]) #[target, x, y, mkNatLit k, eqDecl.toExpr, newMVar]
+          mvarId.assign val
+          let mvarId ← newMVar.mvarId!.tryClear eqDecl.fvarId
+          return some mvarId
         let rec injection (a b : Expr) := do
+          if let some mvarId ← injectionOffset? a b then
+            return some { mvarId, numNewEqs := 1, subst }
           if (← isConstructorApp a <&&> isConstructorApp b) then
             /- ctor_i ... = ctor_j ... -/
             match (← injectionCore mvarId eqFVarId) with

src/Lean/MetavarContext.lean

@@ -360,8 +360,11 @@ def MetavarContext.getExprAssignmentCore? (m : MetavarContext) (mvarId : MVarId)
 def getExprMVarAssignment? [Monad m] [MonadMCtx m] (mvarId : MVarId) : m (Option Expr) :=
   return (← getMCtx).getExprAssignmentCore? mvarId
 
+def MetavarContext.getDelayedMVarAssignmentCore? (mctx : MetavarContext) (mvarId : MVarId) : Option DelayedMetavarAssignment :=
+  mctx.dAssignment.find? mvarId
+
 def getDelayedMVarAssignment? [Monad m] [MonadMCtx m] (mvarId : MVarId) : m (Option DelayedMetavarAssignment) :=
-  return (← getMCtx).dAssignment.find? mvarId
+  return (← getMCtx).getDelayedMVarAssignmentCore? mvarId
 
 /-- Given a sequence of delayed assignments
    ```

src/Lean/Parser/Basic.lean

@@ -558,6 +558,8 @@ partial def whitespace : ParserFn := fun c s =>
     let curr := input.get' i h
     if curr == '\t' then
       s.mkUnexpectedError (pushMissing := false) "tabs are not allowed; please configure your editor to expand them"
+    else if curr == '\r' then
+      s.mkUnexpectedError (pushMissing := false) "isolated carriage returns are not allowed"
     else if curr.isWhitespace then whitespace c (s.next' input i h)
     else if curr == '-' then
       let i    := input.next' i h
@@ -621,9 +623,8 @@ def hexDigitFn : ParserFn := fun c s =>
 /--
 Parses the whitespace after the `\` when there is a string gap.
 Raises an error if the whitespace does not contain exactly one newline character.
-Processes `\r\n` as a newline.
 -/
-partial def stringGapFn (seenNewline afterCR : Bool) : ParserFn := fun c s =>
+partial def stringGapFn (seenNewline : Bool) : ParserFn := fun c s =>
   let i := s.pos
   if h : c.input.atEnd i then s -- let strLitFnAux handle the EOI error if !seenNewline
   else
@@ -633,13 +634,9 @@ partial def stringGapFn (seenNewline afterCR : Bool) : ParserFn := fun c s =>
         -- Having more than one newline in a string gap is visually confusing
         s.mkUnexpectedError "unexpected additional newline in string gap"
       else
-        stringGapFn true false c (s.next' c.input i h)
-    else if curr == '\r' then
-      stringGapFn seenNewline true c (s.next' c.input i h)
-    else if afterCR then
-      s.mkUnexpectedError "expecting newline after carriage return"
+        stringGapFn true c (s.next' c.input i h)
     else if curr.isWhitespace then
-      stringGapFn seenNewline false c (s.next' c.input i h)
+      stringGapFn seenNewline c (s.next' c.input i h)
     else if seenNewline then
       s
     else
@@ -663,8 +660,8 @@ def quotedCharCoreFn (isQuotable : Char → Bool) (inString : Bool) : ParserFn :
       andthenFn hexDigitFn hexDigitFn c (s.next' input i h)
     else if curr == 'u' then
       andthenFn hexDigitFn (andthenFn hexDigitFn (andthenFn hexDigitFn hexDigitFn)) c (s.next' input i h)
-    else if inString && (curr == '\n' || curr == '\r') then
-      stringGapFn false false c s
+    else if inString && curr == '\n' then
+      stringGapFn false c s
     else
       s.mkUnexpectedError "invalid escape sequence"
 

src/Lean/Parser/Extension.lean

@@ -437,11 +437,12 @@ def getSyntaxNodeKinds (env : Environment) : List SyntaxNodeKind :=
 def getTokenTable (env : Environment) : TokenTable :=
   (parserExtension.getState env).tokens
 
-def mkInputContext (input : String) (fileName : String) : InputContext := {
-  input    := input,
-  fileName := fileName,
-  fileMap  := input.toFileMap
-}
+-- Note: `crlfToLf` preserves logical line and column numbers for each character.
+def mkInputContext (input : String) (fileName : String) (normalizeLineEndings := true) : InputContext :=
+  let input' := if normalizeLineEndings then input.crlfToLf else input
+  { input    := input',
+    fileName := fileName,
+    fileMap  := input'.toFileMap }
 
 def mkParserState (input : String) : ParserState :=
   { cache := initCacheForInput input }
@@ -453,7 +454,7 @@ def runParserCategory (env : Environment) (catName : Name) (input : String) (fil
   let s := p.run ictx { env, options := {} } (getTokenTable env) (mkParserState input)
   if !s.allErrors.isEmpty  then
     Except.error (s.toErrorMsg ictx)
-  else if input.atEnd s.pos then
+  else if ictx.input.atEnd s.pos then
     Except.ok s.stxStack.back
   else
     Except.error ((s.mkError "end of input").toErrorMsg ictx)

src/Lean/Parser/Module.lean

@@ -141,7 +141,7 @@ partial def testParseModuleAux (env : Environment) (inputCtx : InputContext) (s
     match parseCommand inputCtx { env := env, options := {} } state msgs with
     | (stx, state, msgs) =>
       if isTerminalCommand stx then
-        if msgs.isEmpty then
+        if !msgs.hasUnreported then
           pure stxs
         else do
           msgs.forM fun msg => msg.toString >>= IO.println

src/Lean/PrettyPrinter.lean

@@ -110,16 +110,14 @@ namespace MessageData
 open Lean PrettyPrinter Delaborator
 
 /--
-Turns a `MetaM FormatWithInfos` into a `MessageData` using `.ofPPFormat` and running the monadic value in the given context.
-Uses the `pp.tagAppFns` option to annotate constants with terminfo, which is necessary for seeing the type on mouse hover.
+Turns a `MetaM FormatWithInfos` into a `MessageData.lazy` which will run the monadic value.
+Uses the `pp.tagAppFns` option to annotate constants with terminfo,
+which is necessary for seeing the type on mouse hover.
 -/
-def ofFormatWithInfos
-    (fmt : MetaM FormatWithInfos)
-    (noContext : Unit → Format := fun _ => "<no context, could not generate MessageData>") : MessageData :=
-  .ofPPFormat
-  { pp := fun
-      | some ctx => ctx.runMetaM <| withOptions (pp.tagAppFns.set · true) fmt
-      | none => return noContext () }
+def ofFormatWithInfosM (fmt : MetaM FormatWithInfos) : MessageData :=
+  .lazy fun ctx => ctx.runMetaM <|
+    withOptions (pp.tagAppFns.set · true) <|
+      .ofFormatWithInfos <$> fmt
 
 /-- Pretty print a const expression using `delabConst` and generate terminfo.
 This function avoids inserting `@` if the constant is for a function whose first
@@ -127,13 +125,13 @@ argument is implicit, which is what the default `toMessageData` for `Expr` does.
 Panics if `e` is not a constant. -/
 def ofConst (e : Expr) : MessageData :=
   if e.isConst then
-    .ofFormatWithInfos (PrettyPrinter.ppExprWithInfos (delab := delabConst) e) fun _ => f!"{e}"
+    .ofFormatWithInfosM (PrettyPrinter.ppExprWithInfos (delab := delabConst) e)
   else
     panic! "not a constant"
 
 /-- Generates `MessageData` for a declaration `c` as `c.{<levels>} <params> : <type>`, with terminfo. -/
 def signature (c : Name) : MessageData :=
-  .ofFormatWithInfos (PrettyPrinter.ppSignature c) fun _ => f!"{c}"
+  .ofFormatWithInfosM (PrettyPrinter.ppSignature c)
 
 end MessageData
 

src/Lean/Server/FileWorker.lean

@@ -196,7 +196,7 @@ This option can only be set on the command line, not in the lakefile or via `set
         return .pure ()
   where
     go (node : SnapshotTree) (st : ReportSnapshotsState) : BaseIO (Task ReportSnapshotsState) := do
-      if !node.element.diagnostics.msgLog.isEmpty then
+      if node.element.diagnostics.msgLog.hasUnreported then
         let diags ←
           if let some memorized ← node.element.diagnostics.interactiveDiagsRef?.bindM fun ref => do
               return (← ref.get).bind (·.get? MemorizedInteractiveDiagnostics) then
@@ -705,12 +705,9 @@ def initAndRunWorker (i o e : FS.Stream) (opts : Options) : IO UInt32 := do
   let initParams ← i.readLspRequestAs "initialize" InitializeParams
   let ⟨_, param⟩ ← i.readLspNotificationAs "textDocument/didOpen" LeanDidOpenTextDocumentParams
   let doc := param.textDocument
-  /- NOTE(WN): `toFileMap` marks line beginnings as immediately following
-    "\n", which should be enough to handle both LF and CRLF correctly.
-    This is because LSP always refers to characters by (line, column),
-    so if we get the line number correct it shouldn't matter that there
-    is a CR there. -/
-  let meta : DocumentMeta := ⟨doc.uri, doc.version, doc.text.toFileMap, param.dependencyBuildMode?.getD .always⟩
+  /- Note (kmill): LSP always refers to characters by (line, column),
+     so converting CRLF to LF preserves line and column numbers. -/
+  let meta : DocumentMeta := ⟨doc.uri, doc.version, doc.text.crlfToLf.toFileMap, param.dependencyBuildMode?.getD .always⟩
   let e := e.withPrefix s!"[{param.textDocument.uri}] "
   let _ ← IO.setStderr e
   let (ctx, st) ← try

src/Lean/Server/FileWorker/Utils.lean

@@ -15,22 +15,6 @@ namespace Lean.Server.FileWorker
 open Snapshots
 open IO
 
-structure CancelToken where
-  ref : IO.Ref Bool
-
-namespace CancelToken
-
-def new : IO CancelToken :=
-  CancelToken.mk <$> IO.mkRef false
-
-def set (tk : CancelToken) : BaseIO Unit :=
-  tk.ref.set true
-
-def isSet (tk : CancelToken) : BaseIO Bool :=
-  tk.ref.get
-
-end CancelToken
-
 -- TEMP: translate from new heterogeneous snapshot tree to old homogeneous async list
 private partial def mkCmdSnaps (initSnap : Language.Lean.InitialSnapshot) :
     AsyncList IO.Error Snapshot := Id.run do
@@ -49,7 +33,7 @@ where
         stx := cmdParsed.data.stx
         mpState := cmdParsed.data.parserState
         cmdState := finished.cmdState
-      } (match cmdParsed.next? with
+      } (match cmdParsed.nextCmdSnap? with
         | some next => .delayed <| next.task.bind go
         | none => .nil)
 

src/Lean/Server/Snapshots.lean

@@ -59,6 +59,7 @@ def runCommandElabM (snap : Snapshot) (meta : DocumentMeta) (c : CommandElabM α
     fileMap := meta.text,
     tacticCache? := none
     snap? := none
+    cancelTk? := none
   }
   c.run ctx |>.run' snap.cmdState
 

src/Lean/Server/Utils.lean

@@ -75,7 +75,10 @@ structure DocumentMeta where
   uri                 : Lsp.DocumentUri
   /-- Version number of the document. Incremented whenever the document is edited. -/
   version             : Nat
-  /-- Current text of the document. -/
+  /--
+  Current text of the document.
+  It is maintained such that it is normalized using `String.crlfToLf`, which preserves logical line/column numbers.
+  (Note: we assume that edit operations never split or merge `\r\n` line endings.) -/
   text                : FileMap
   /--
   Controls when dependencies of the document are built on `textDocument/didOpen` notifications.
@@ -98,7 +101,11 @@ def replaceLspRange (text : FileMap) (r : Lsp.Range) (newText : String) : FileMa
   let «end» := text.lspPosToUtf8Pos r.«end»
   let pre := text.source.extract 0 start
   let post := text.source.extract «end» text.source.endPos
-  (pre ++ newText ++ post).toFileMap
+  -- `pre` and `post` already have normalized line endings, so only `newText` needs its endings normalized.
+  -- Note: this assumes that editing never separates a `\r\n`.
+  -- If `pre` ends with `\r` and `newText` begins with `\n`, the result is potentially inaccurate.
+  -- If this is ever a problem, we could store a second unnormalized FileMap, edit it, and normalize it here.
+  (pre ++ newText.crlfToLf ++ post).toFileMap
 
 open IO
 
@@ -125,7 +132,7 @@ def applyDocumentChange (oldText : FileMap) : (change : Lsp.TextDocumentContentC
   | TextDocumentContentChangeEvent.rangeChange (range : Range) (newText : String) =>
     replaceLspRange oldText range newText
   | TextDocumentContentChangeEvent.fullChange (newText : String) =>
-    newText.toFileMap
+    newText.crlfToLf.toFileMap
 
 /-- Returns the document contents with all changes applied. -/
 def foldDocumentChanges (changes : Array Lsp.TextDocumentContentChangeEvent) (oldText : FileMap) : FileMap :=

src/Lean/Server/Watchdog.lean

@@ -728,12 +728,9 @@ end RequestHandling
 section NotificationHandling
   def handleDidOpen (p : LeanDidOpenTextDocumentParams) : ServerM Unit :=
     let doc := p.textDocument
-    /- NOTE(WN): `toFileMap` marks line beginnings as immediately following
-       "\n", which should be enough to handle both LF and CRLF correctly.
-       This is because LSP always refers to characters by (line, column),
-       so if we get the line number correct it shouldn't matter that there
-       is a CR there. -/
-    startFileWorker ⟨doc.uri, doc.version, doc.text.toFileMap, p.dependencyBuildMode?.getD .always⟩
+    /- Note (kmill): LSP always refers to characters by (line, column),
+      so converting CRLF to LF preserves line and column numbers. -/
+    startFileWorker ⟨doc.uri, doc.version, doc.text.crlfToLf.toFileMap, p.dependencyBuildMode?.getD .always⟩
 
   def handleDidChange (p : DidChangeTextDocumentParams) : ServerM Unit := do
     let doc := p.textDocument

src/Lean/Syntax.lean

@@ -30,6 +30,9 @@ def SourceInfo.updateTrailing (trailing : Substring) : SourceInfo → SourceInfo
   | SourceInfo.original leading pos _ endPos => SourceInfo.original leading pos trailing endPos
   | info                                     => info
 
+def SourceInfo.getRange? (canonicalOnly := false) (info : SourceInfo) : Option String.Range :=
+  return ⟨(← info.getPos? canonicalOnly), (← info.getTailPos? canonicalOnly)⟩
+
 /-! # Syntax AST -/
 
 inductive IsNode : Syntax → Prop where
@@ -80,6 +83,34 @@ end SyntaxNode
 
 namespace Syntax
 
+/--
+Compare syntax structures and position ranges, but not whitespace.
+We generally assume that if syntax trees equal in this way generate the same elaboration output,
+including positions contained in e.g. diagnostics and the info tree.
+-/
+partial def structRangeEq : Syntax → Syntax → Bool
+  | .missing, .missing => true
+  | .node info k args, .node info' k' args' =>
+    info.getRange? == info'.getRange? && k == k' && args.isEqv args' structRangeEq
+  | .atom info val, .atom info' val' => info.getRange? == info'.getRange? && val == val'
+  | .ident info rawVal val preresolved, .ident info' rawVal' val' preresolved' =>
+    info.getRange? == info'.getRange? && rawVal == rawVal' && val == val' &&
+    preresolved == preresolved'
+  | _, _ => false
+
+/-- Like `structRangeEq` but prints trace on failure if `trace.Elab.reuse` is activated. -/
+def structRangeEqWithTraceReuse (opts : Options) (stx1 stx2 : Syntax) : Bool :=
+  if stx1.structRangeEq stx2 then
+    true
+  else
+    if opts.getBool `trace.Elab.reuse then
+      dbg_trace "reuse stopped:
+{stx1.formatStx (showInfo := true)} !=
+{stx2.formatStx (showInfo := true)}"
+      false
+    else
+      false
+
 def getAtomVal : Syntax → String
   | atom _ val => val
   | _          => ""
@@ -187,13 +218,6 @@ partial def updateTrailing (trailing : Substring) : Syntax → Syntax
      Syntax.node info k args
   | s => s
 
-partial def getTailWithPos : Syntax → Option Syntax
-  | stx@(atom info _)   => info.getPos?.map fun _ => stx
-  | stx@(ident info ..) => info.getPos?.map fun _ => stx
-  | node SourceInfo.none _ args => args.findSomeRev? getTailWithPos
-  | stx@(node ..) => stx
-  | _ => none
-
 open SourceInfo in
 /-- Split an `ident` into its dot-separated components while preserving source info.
 Macro scopes are first erased.  For example, `` `foo.bla.boo._@._hyg.4 `` ↦ `` [`foo, `bla, `boo] ``.

src/Lean/Util/PPExt.lean

@@ -40,6 +40,10 @@ structure PPContext where
   openDecls     : List OpenDecl := []
 
 abbrev PrettyPrinter.InfoPerPos := RBMap Nat Elab.Info compare
+/-- A format tree with `Elab.Info` annotations.
+Each `.tag n _` node is annotated with `infos[n]`.
+This is used to attach semantic data such as expressions
+to pretty-printer outputs. -/
 structure FormatWithInfos where
   fmt : Format
   infos : PrettyPrinter.InfoPerPos

src/Lean/Widget/InteractiveDiagnostic.lean

@@ -128,10 +128,8 @@ where
   }
 
   go (nCtx : NamingContext) : Option MessageDataContext → MessageData → MsgFmtM Format
-  | _,         ofFormat fmt             => withIgnoreTags fmt
-  | none,      ofPPFormat fmt           => (·.fmt) <$> fmt.pp none
-  | some ctx,  ofPPFormat fmt           => do
-    let ⟨fmt, infos⟩ ← fmt.pp (mkPPContext nCtx ctx)
+  | none,      ofFormatWithInfos ⟨fmt, _⟩ => withIgnoreTags fmt
+  | some ctx,  ofFormatWithInfos ⟨fmt, infos⟩ => do
     let t ← pushEmbed <| EmbedFmt.code (mkContextInfo nCtx ctx) infos
     return Format.tag t fmt
   | none,      ofGoal mvarId            => pure $ "goal " ++ format (mkMVar mvarId)
@@ -162,6 +160,11 @@ where
         pure (.strict (← children.mapM (go nCtx ctx)))
     let e := .trace data.cls header data.collapsed nodes
     return .tag (← pushEmbed e) ".\n"
+  | ctx?,     ofLazy f _          => do
+    let dyn ← f (ctx?.map (mkPPContext nCtx))
+    let some msg := dyn.get? MessageData
+      | throw <| IO.userError "MessageData.ofLazy: expected MessageData in Dynamic"
+    go nCtx ctx? msg
 
   /-- Recursively moves child nodes after the first `blockSize` into a new "more" node. -/
   chopUpChildren (cls : Name) (blockSize : Nat) (children : Subarray MessageData) :

src/lake/Lake/Build/Actions.lean

@@ -5,7 +5,7 @@ Authors: Gabriel Ebner, Sebastian Ullrich, Mac Malone, Siddharth Bhat
 -/
 import Lake.Util.Proc
 import Lake.Util.NativeLib
-import Lake.Build.Basic
+import Lake.Build.Job
 
 /-! # Common Build Actions
 Low level actions to build common Lean artifacts via the Lean toolchain.
@@ -115,7 +115,7 @@ def download  (url : String) (file : FilePath) : LogIO PUnit := do
     createParentDirs file
   proc (quiet := true) {
     cmd := "curl"
-    args := #["-f", "-o", file.toString, "-L", url]
+    args := #["-s", "-S", "-f", "-o", file.toString, "-L", url]
   }
 
 /-- Unpack an archive `file` using `tar` into the directory `dir`. -/

src/lake/Lake/Build/Basic.lean

@@ -5,7 +5,6 @@ Authors: Mac Malone
 -/
 import Lake.Util.Log
 import Lake.Util.Exit
-import Lake.Util.Task
 import Lake.Util.Lift
 import Lake.Config.Context
 import Lake.Build.Trace
@@ -52,96 +51,17 @@ Whether the build should show progress information.
 def BuildConfig.showProgress (cfg : BuildConfig) : Bool :=
   (cfg.noBuild ∧ cfg.verbosity == .verbose) ∨ cfg.verbosity != .quiet
 
-/-- Information on what this job did. -/
-inductive JobAction
-/-- No information about this job's action is available. -/
-| unknown
-/-- Tried to replay a cached build action (set by `buildFileUnlessUpToDate`) -/
-| replay
-/-- Tried to fetch a build from a store (can be set by `buildUnlessUpToDate?`) -/
-| fetch
-/-- Tried to perform a build action (set by `buildUnlessUpToDate?`) -/
-| build
-deriving Inhabited, Repr, DecidableEq, Ord
-
-instance : LT JobAction := ltOfOrd
-instance : LE JobAction := leOfOrd
-instance : Min JobAction := minOfLe
-instance : Max JobAction := maxOfLe
-
-@[inline] def JobAction.merge (a b : JobAction) : JobAction :=
-  max a b
-
-def JobAction.verb (failed : Bool) : JobAction → String
-| .unknown => if failed then "Running" else "Ran"
-| .replay => if failed then "Replaying" else "Replayed"
-| .fetch => if failed then "Fetching" else "Fetched"
-| .build => if failed then "Building" else "Built"
-
-/-- Mutable state of a Lake job. -/
-structure JobState where
-  /-- The job's log. -/
-  log : Log := {}
-  /-- Tracks whether this job performed any significant build action. -/
-  action : JobAction := .unknown
-
-/--
-Resets the job state after a checkpoint (e.g., registering the job).
-Preserves state that downstream jobs want to depend on while resetting
-job-local state that should not be inherited by downstream jobs.
--/
-@[inline] def JobState.renew (_ : JobState) : JobState where
-  log := {}
-  action := .unknown
-
-def JobState.merge (a b : JobState) : JobState where
-  log := a.log ++ b.log
-  action := a.action.merge b.action
-
-@[inline] def JobState.modifyLog (f : Log → Log) (s : JobState) :=
-  {s with log := f s.log}
-
-/-- The result of a Lake job. -/
-abbrev JobResult α := EResult Log.Pos JobState α
-
-/-- The `Task` of a Lake job. -/
-abbrev JobTask α := BaseIOTask (JobResult α)
-
-/-- A Lake job. -/
-structure Job (α : Type u)  where
-  task : JobTask α
-  caption : String
+/-- A Lake job with an opaque value type in `Type`. -/
+declare_opaque_type OpaqueJob
 
 /-- A Lake context with a build configuration and additional build data. -/
 structure BuildContext extends BuildConfig, Context where
   leanTrace : BuildTrace
-  registeredJobs : IO.Ref (Array (Job Unit))
+  registeredJobs : IO.Ref (Array OpaqueJob)
 
 /-- A transformer to equip a monad with a `BuildContext`. -/
 abbrev BuildT := ReaderT BuildContext
 
-/-- The monad of asynchronous Lake jobs. Lifts into `FetchM`. -/
-abbrev JobM := BuildT <| EStateT Log.Pos JobState BaseIO
-
-instance [Pure m] : MonadLift LakeM (BuildT m) where
-  monadLift x := fun ctx => pure <| x.run ctx.toContext
-
-instance : MonadStateOf Log JobM where
-  get := (·.log) <$> get
-  set log := modify fun s => {s with log}
-  modifyGet f := modifyGet fun s => let (a, log) := f s.log; (a, {s with log})
-
-instance : MonadStateOf JobState JobM := inferInstance
-
-instance : MonadLog JobM := .ofMonadState
-instance : MonadError JobM := ELog.monadError
-instance : Alternative JobM := ELog.alternative
-instance : MonadLift LogIO JobM := ⟨ELogT.takeAndRun⟩
-
-/-- Record that this job is trying to perform some action. -/
-@[inline] def updateAction (action : JobAction) : JobM PUnit :=
-  modify fun s => {s with action := s.action.merge action}
-
 /-- A monad equipped with a Lake build context. -/
 abbrev MonadBuild (m : Type → Type u) :=
   MonadReaderOf BuildContext m
@@ -175,19 +95,3 @@ abbrev MonadBuild (m : Type → Type u) :=
 
 /-- The internal core monad of Lake builds.  Not intended for user use. -/
 abbrev CoreBuildM := BuildT LogIO
-
-/-- The monad used to spawn asynchronous Lake build jobs. Lifts into `FetchM`. -/
-abbrev SpawnM := BuildT BaseIO
-
-/-- The monad used to spawn asynchronous Lake build jobs. **Replaced by `SpawnM`.** -/
-@[deprecated SpawnM] abbrev SchedulerM := SpawnM
-
-/--
-Logs a build step with `message`.
-
-**Deprecated:**  Build steps are now managed by a top-level build monitor.
-As a result, this no longer functions the way it used to. It now just logs the
-`message` via `logVerbose`.
--/
-@[deprecated] def logStep (message : String) : JobM Unit := do
-  logVerbose message

src/lake/Lake/Build/Common.lean

@@ -71,8 +71,9 @@ of the `traceFile`. If rebuilt, save the new `depTrace` to the `tracefile`.
 @[inline] def buildUnlessUpToDate
   [CheckExists ι] [GetMTime ι] (info : ι)
   (depTrace : BuildTrace) (traceFile : FilePath) (build : JobM PUnit)
+  (action : JobAction := .build) (oldTrace := depTrace)
 : JobM PUnit := do
-  discard <| buildUnlessUpToDate? info depTrace traceFile build
+  discard <| buildUnlessUpToDate? info depTrace traceFile build action oldTrace
 
 /-- Fetch the trace of a file that may have its hash already cached in a `.hash` file. -/
 def fetchFileTrace (file : FilePath) : JobM BuildTrace := do

src/lake/Lake/Build/Fetch.lean

@@ -81,32 +81,35 @@ def ensureJob (x : FetchM (Job α))
     let (log, jobLog) := log.split iniPos
     return (.ok (.error jobLog) log, store)
 
+/--
+Registers the job for the top-level build monitor,
+(e.g., the Lake CLI progress UI), assigning it `caption`.
+-/
+def registerJob (caption : String) (job : Job α) : FetchM (Job α) := do
+  let job := job.setCaption caption
+  (← getBuildContext).registeredJobs.modify (·.push job)
+  return job.renew
+
 /--
 Registers the produced job for the top-level build monitor
 (e.g., the Lake CLI progress UI), assigning it `caption`.
 
 Stray I/O, logs, and errors produced by `x` will be wrapped into the job.
 -/
-@[inline] def withRegisterJob
+def withRegisterJob
   (caption : String) (x : FetchM (Job α))
 : FetchM (Job α) := do
   let job ← ensureJob x
-  let job := job.setCaption caption
-  let regJob := job.mapResult (sync := true) discard
-  (← readThe BuildContext).registeredJobs.modify (·.push regJob)
-  return job.renew
+  registerJob caption job
 
 /--
 Registers the produced job for the top-level build monitor
 if it is not already (i.e., it has an empty caption).
 -/
 @[inline] def maybeRegisterJob
-  (fallbackCaption : String) (job : Job α)
+  (caption : String) (job : Job α)
 : FetchM (Job α) := do
   if job.caption.isEmpty then
-    let job := job.setCaption fallbackCaption
-    let regJob := job.mapResult (sync := true) discard
-    (← readThe BuildContext).registeredJobs.modify (·.push regJob)
-    return job.renew
+    registerJob caption job
   else
     return job

src/lake/Lake/Build/Job.lean

@@ -3,13 +3,136 @@ Copyright (c) 2022 Mac Malone. All rights reserved.
 Released under Apache 2.0 license as described in the file LICENSE.
 Authors: Mac Malone
 -/
-import Lake.Build.Trace
+import Lake.Util.Task
 import Lake.Build.Basic
 
 open System
 
 namespace Lake
 
+/-- Information on what this job did. -/
+inductive JobAction
+/-- No information about this job's action is available. -/
+| unknown
+/-- Tried to replay a cached build action (set by `buildFileUnlessUpToDate`) -/
+| replay
+/-- Tried to fetch a build from a store (can be set by `buildUnlessUpToDate?`) -/
+| fetch
+/-- Tried to perform a build action (set by `buildUnlessUpToDate?`) -/
+| build
+deriving Inhabited, Repr, DecidableEq, Ord
+
+instance : LT JobAction := ltOfOrd
+instance : LE JobAction := leOfOrd
+instance : Min JobAction := minOfLe
+instance : Max JobAction := maxOfLe
+
+@[inline] def JobAction.merge (a b : JobAction) : JobAction :=
+  max a b
+
+def JobAction.verb (failed : Bool) : JobAction → String
+| .unknown => if failed then "Running" else "Ran"
+| .replay => if failed then "Replaying" else "Replayed"
+| .fetch => if failed then "Fetching" else "Fetched"
+| .build => if failed then "Building" else "Built"
+
+/-- Mutable state of a Lake job. -/
+structure JobState where
+  /-- The job's log. -/
+  log : Log := {}
+  /-- Tracks whether this job performed any significant build action. -/
+  action : JobAction := .unknown
+  deriving Inhabited
+
+/--
+Resets the job state after a checkpoint (e.g., registering the job).
+Preserves state that downstream jobs want to depend on while resetting
+job-local state that should not be inherited by downstream jobs.
+-/
+@[inline] def JobState.renew (_ : JobState) : JobState where
+  log := {}
+  action := .unknown
+
+def JobState.merge (a b : JobState) : JobState where
+  log := a.log ++ b.log
+  action := a.action.merge b.action
+
+@[inline] def JobState.modifyLog (f : Log → Log) (s : JobState) :=
+  {s with log := f s.log}
+
+/-- The result of a Lake job. -/
+abbrev JobResult α := EResult Log.Pos JobState α
+
+/-- The `Task` of a Lake job. -/
+abbrev JobTask α := BaseIOTask (JobResult α)
+
+/-- The monad of asynchronous Lake jobs. Lifts into `FetchM`. -/
+abbrev JobM := BuildT <| EStateT Log.Pos JobState BaseIO
+
+instance [Pure m] : MonadLift LakeM (BuildT m) where
+  monadLift x := fun ctx => pure <| x.run ctx.toContext
+
+instance : MonadStateOf Log JobM where
+  get := (·.log) <$> get
+  set log := modify fun s => {s with log}
+  modifyGet f := modifyGet fun s => let (a, log) := f s.log; (a, {s with log})
+
+instance : MonadStateOf JobState JobM := inferInstance
+
+instance : MonadLog JobM := .ofMonadState
+instance : MonadError JobM := ELog.monadError
+instance : Alternative JobM := ELog.alternative
+instance : MonadLift LogIO JobM := ⟨ELogT.takeAndRun⟩
+
+/-- Record that this job is trying to perform some action. -/
+@[inline] def updateAction (action : JobAction) : JobM PUnit :=
+  modify fun s => {s with action := s.action.merge action}
+
+/-- The monad used to spawn asynchronous Lake build jobs. Lifts into `FetchM`. -/
+abbrev SpawnM := BuildT BaseIO
+
+/-- The monad used to spawn asynchronous Lake build jobs. **Replaced by `SpawnM`.** -/
+@[deprecated SpawnM] abbrev SchedulerM := SpawnM
+
+/--
+Logs a build step with `message`.
+
+**Deprecated:**  Build steps are now managed by a top-level build monitor.
+As a result, this no longer functions the way it used to. It now just logs the
+`message` via `logVerbose`.
+-/
+@[deprecated] def logStep (message : String) : JobM Unit := do
+  logVerbose message
+
+/-- A Lake job. -/
+structure Job (α : Type u)  where
+  task : JobTask α
+  caption : String
+  deriving Inhabited
+
+structure BundledJob where
+  {type : Type u}
+  job : Job type
+  deriving Inhabited
+
+instance : CoeOut (Job α) BundledJob := ⟨.mk⟩
+
+hydrate_opaque_type OpaqueJob BundledJob
+
+abbrev OpaqueJob.type (job : OpaqueJob) : Type :=
+  job.get.type
+
+abbrev OpaqueJob.toJob (job : OpaqueJob) : Job job.type :=
+  job.get.job
+
+abbrev OpaqueJob.task (job : OpaqueJob) : JobTask job.type :=
+  job.toJob.task
+
+abbrev OpaqueJob.caption (job : OpaqueJob) : String :=
+  job.toJob.caption
+
+instance : CoeDep OpaqueJob job (Job job.type) := ⟨job.toJob⟩
+
 namespace Job
 
 @[inline] def ofTask (task : JobTask α) (caption := "") : Job α :=

src/lake/Lake/Build/Package.lean

@@ -35,10 +35,12 @@ def Package.recBuildExtraDepTargets (self : Package) : FetchM (BuildJob Unit) :=
     job := job.mix <| ← dep.extraDep.fetch
   -- Fetch pre-built release if desired and this package is a dependency
   if self.name ≠ (← getWorkspace).root.name ∧ self.preferReleaseBuild then
-    job := job.add <| ← (← self.optRelease.fetch).bindSync fun success t => do
-      unless success do
-        logWarning "failed to fetch cloud release; falling back to local build"
-      return ((), t)
+    job := job.add <| ←
+      withRegisterJob s!"{self.name}:optRelease" do
+        (← self.optRelease.fetch).bindSync fun success t => do
+          unless success do
+            logWarning "failed to fetch cloud release; falling back to local build"
+          return ((), t)
   -- Build this package's extra dep targets
   for target in self.extraDepTargets do
     job := job.mix <| ← self.fetchTargetJob target
@@ -50,26 +52,26 @@ def Package.extraDepFacetConfig : PackageFacetConfig extraDepFacet :=
 
 /-- Download and unpack the package's prebuilt release archive (from GitHub). -/
 def Package.fetchOptRelease (self : Package) : FetchM (BuildJob Bool) := Job.async do
-  updateAction .fetch
   let repo := GitRepo.mk self.dir
   let repoUrl? := self.releaseRepo? <|> self.remoteUrl?
   let some repoUrl := repoUrl? <|> (← repo.getFilteredRemoteUrl?)
     | logInfo s!"{self.name}: wanted prebuilt release, \
-        but package's repository URL was not known; it may need to set 'releaseRepo'"
+        but repository URL not known; the package may need to set 'releaseRepo'"
+      updateAction .fetch
       return (false, .nil)
   let some tag ← repo.findTag?
-    | logInfo s!"{self.name}: wanted prebuilt release, \
-        but could not find an associated tag for the package's revision"
+    | logInfo s!"{self.name}: wanted prebuilt release, but no tag found for revision"
+      updateAction .fetch
       return (false, .nil)
   let url := s!"{repoUrl}/releases/download/{tag}/{self.buildArchive}"
-  let logName := s!"{self.name}/{tag}/{self.buildArchive}"
   let depTrace := Hash.ofString url
   let traceFile := FilePath.mk <| self.buildArchiveFile.toString ++ ".trace"
   let upToDate ← buildUnlessUpToDate? (action := .fetch) self.buildArchiveFile depTrace traceFile do
-    logVerbose s!"downloading {logName}"
+    logVerbose s!"downloading {url}"
     download url self.buildArchiveFile
   unless upToDate && (← self.buildDir.pathExists) do
-    logVerbose s!"unpacking {logName}"
+    updateAction .fetch
+    logVerbose s!"unpacking {self.name}/{tag}/{self.buildArchive}"
     untar self.buildArchiveFile self.buildDir
   return (true, .nil)
 

src/lake/Lake/Build/Run.lean

@@ -25,24 +25,29 @@ def mkBuildContext (ws : Workspace) (config : BuildConfig) : BaseIO BuildContext
     leanTrace := Hash.ofString ws.lakeEnv.leanGithash
   }
 
+/-- Unicode icons that make up the spinner in animation order. -/
+def Monitor.spinnerFrames :=
+  #['⣾','⣷','⣯','⣟','⡿','⢿','⣻','⣽']
+
 /-- Context of the Lake build monitor. -/
 structure MonitorContext where
   totalJobs : Nat
   out : IO.FS.Stream
   outLv : LogLevel
   failLv : LogLevel
-  showProgress : Bool
+  minAction : JobAction
   useAnsi : Bool
+  showProgress : Bool
   /-- How often to poll jobs (in milliseconds). -/
-  updateFrequency : Nat := 100
+  updateFrequency : Nat
 
 /-- State of the Lake build monitor. -/
 structure MonitorState where
   jobNo : Nat := 1
-  jobs : Array (Job Unit)
   failures : Array String
   resetCtrl : String
   lastUpdate : Nat
+  spinnerIdx : Fin Monitor.spinnerFrames.size := ⟨0, by decide⟩
 
 /-- Monad of the Lake build monitor. -/
 abbrev MonitorM := ReaderT MonitorContext <| StateT MonitorState BaseIO
@@ -76,29 +81,35 @@ namespace Monitor
 @[inline] nonrec def flush : MonitorM PUnit := do
   flush (← read).out
 
-def renderProgress : MonitorM PUnit := do
-  let {jobNo, jobs, ..} ← get
+def renderProgress (running unfinished : Array OpaqueJob) (h : 0 < unfinished.size) : MonitorM PUnit := do
+  let {jobNo, ..} ← get
   let {totalJobs, useAnsi, showProgress, ..} ← read
   if showProgress ∧ useAnsi then
-    if h : 0 < jobs.size then
-      let caption := jobs[0]'h |>.caption
-      let resetCtrl ← modifyGet fun s => (s.resetCtrl, {s with resetCtrl := Ansi.resetLine})
-      print s!"{resetCtrl}[{jobNo}/{totalJobs}] Checking {caption}"
-      flush
+    let spinnerIcon ← modifyGet fun s =>
+        (spinnerFrames[s.spinnerIdx], {s with spinnerIdx := s.spinnerIdx + ⟨1, by decide⟩})
+    let resetCtrl ← modifyGet fun s => (s.resetCtrl, {s with resetCtrl := Ansi.resetLine})
+    let caption :=
+      if _ : 0 < running.size then
+        s!"Running {running[0].caption} (+ {running.size - 1} more)"
+      else
+        s!"Running {unfinished[0].caption}"
+    print s!"{resetCtrl}{spinnerIcon} [{jobNo}/{totalJobs}] {caption}"
+    flush
 
-def reportJob (job : Job Unit) : MonitorM PUnit := do
+def reportJob (job : OpaqueJob) : MonitorM PUnit := do
   let {jobNo, ..} ← get
-  let {totalJobs, failLv, outLv, out, useAnsi, showProgress, ..} ← read
-  let {log, action, ..} := job.task.get.state
+  let {totalJobs, failLv, outLv, out, useAnsi, showProgress, minAction, ..} ← read
+  let {task, caption} := job.toJob
+  let {log, action, ..} := task.get.state
   let maxLv := log.maxLv
   let failed := log.hasEntries ∧ maxLv ≥ failLv
   if failed then
-    modify fun s => {s with failures := s.failures.push job.caption}
+    modify fun s => {s with failures := s.failures.push caption}
   let hasOutput := failed ∨ (log.hasEntries ∧ maxLv ≥ outLv)
-  if hasOutput ∨ (showProgress ∧ action ≥ .fetch) then
+  if hasOutput ∨ (showProgress ∧ ¬ useAnsi ∧ action ≥ minAction) then
     let verb := action.verb failed
     let icon := if hasOutput then maxLv.icon else '✔'
-    let caption := s!"{icon} [{jobNo}/{totalJobs}] {verb} {job.caption}"
+    let caption := s!"{icon} [{jobNo}/{totalJobs}] {verb} {caption}"
     let caption :=
       if useAnsi then
         let color := if hasOutput then maxLv.ansiColor else "32"
@@ -112,33 +123,36 @@ def reportJob (job : Job Unit) : MonitorM PUnit := do
       log.replay (logger := .stream out outLv useAnsi)
     flush
 
-def pollJobs : MonitorM PUnit := do
-  let prevJobs ← modifyGet fun s => (s.jobs, {s with jobs := #[]})
-  for h : i in [0:prevJobs.size] do
-    let job := prevJobs[i]'h.upper
-    if (← IO.hasFinished job.task) then
+def poll (jobs : Array OpaqueJob): MonitorM (Array OpaqueJob × Array OpaqueJob) := do
+  jobs.foldlM (init := (#[], #[])) fun (running, unfinished) job => do
+    match (← IO.getTaskState job.task) with
+    | .finished =>
       reportJob job
       modify fun s => {s with jobNo := s.jobNo + 1}
-    else
-      modify fun s => {s with jobs := s.jobs.push job}
+      return (running, unfinished)
+    | .running =>
+      return (running.push job, unfinished.push job)
+    | .waiting =>
+      return (running, unfinished.push job)
 
 def sleep : MonitorM PUnit := do
   let now ← IO.monoMsNow
-  let lastUpdate ← modifyGet fun s => (s.lastUpdate, {s with lastUpdate := now})
+  let lastUpdate := (← get).lastUpdate
   let sleepTime : Nat := (← read).updateFrequency - (now - lastUpdate)
   if sleepTime > 0 then
     IO.sleep sleepTime.toUInt32
+  let now ← IO.monoMsNow
+  modify fun s => {s with lastUpdate := now}
 
-partial def loop : MonitorM PUnit := do
-  renderProgress
-  pollJobs
-  if 0 < (← get).jobs.size then
-    renderProgress
+partial def loop (jobs : Array OpaqueJob) : MonitorM PUnit := do
+  let (running, unfinished) ← poll jobs
+  if h : 0 < unfinished.size then
+    renderProgress running unfinished h
     sleep
-    loop
+    loop unfinished
 
-def main : MonitorM PUnit := do
-  loop
+def main (jobs : Array OpaqueJob) : MonitorM PUnit := do
+  loop jobs
   let resetCtrl ← modifyGet fun s => (s.resetCtrl, {s with resetCtrl := ""})
   unless resetCtrl.isEmpty do
     print resetCtrl
@@ -148,9 +162,10 @@ end Monitor
 
 /-- The job monitor function. An auxiliary definition for `runFetchM`. -/
 def monitorJobs
-  (jobs : Array (Job Unit))
+  (jobs : Array OpaqueJob)
   (out : IO.FS.Stream)
   (failLv outLv : LogLevel)
+  (minAction : JobAction)
   (useAnsi showProgress : Bool)
   (resetCtrl : String := "")
   (initFailures : Array String := #[])
@@ -158,15 +173,15 @@ def monitorJobs
   (updateFrequency := 100)
 : BaseIO (Array String) := do
   let ctx := {
-    totalJobs, out, failLv, outLv,
+    totalJobs, out, failLv, outLv, minAction
     useAnsi, showProgress, updateFrequency
   }
   let s := {
-    jobs, resetCtrl
+    resetCtrl
     lastUpdate := ← IO.monoMsNow
     failures := initFailures
   }
-  let (_,s) ← Monitor.main.run ctx s
+  let (_,s) ← Monitor.main jobs |>.run ctx s
   return s.failures
 
 /--
@@ -188,7 +203,7 @@ def Workspace.runFetchM
   -- Job Computation
   let caption := "Computing build jobs"
   if showAnsiProgress then
-    print! out s!"[?/?] {caption}"
+    print! out s!"⣿ [?/?] {caption}"
     flush out
   let (a?, log) ← ((withLoggedIO build).run.run'.run ctx).run?
   let failed := log.hasEntries ∧ log.maxLv ≥ failLv
@@ -211,7 +226,8 @@ def Workspace.runFetchM
   -- Job Monitor
   let jobs ← ctx.registeredJobs.get
   let resetCtrl := if showAnsiProgress then Ansi.resetLine else ""
-  let failures ← monitorJobs jobs out failLv outLv useAnsi showProgress
+  let minAction := if cfg.verbosity = .verbose then .unknown else .fetch
+  let failures ← monitorJobs jobs out failLv outLv minAction useAnsi showProgress
     (resetCtrl := resetCtrl) (initFailures := failures)
   -- Failure Report
   if failures.isEmpty then

src/lake/Lake/Build/Trace.lean

@@ -3,7 +3,6 @@ Copyright (c) 2021 Mac Malone. All rights reserved.
 Released under Apache 2.0 license as described in the file LICENSE.
 Authors: Mac Malone
 -/
-import Lake.Util.Newline
 
 open System
 namespace Lake
@@ -135,7 +134,7 @@ instance : ComputeHash FilePath IO := ⟨computeFileHash⟩
 
 def computeTextFileHash (file : FilePath) : IO Hash := do
   let text ← IO.FS.readFile file
-  let text := crlf2lf text
+  let text := text.crlfToLf
   return Hash.ofString text
 
 /--

src/lake/Lake/CLI/Init.lean

@@ -148,6 +148,8 @@ git = \"https://github.com/leanprover-community/mathlib4.git\"
 name = {repr libRoot}
 "
 
+def readmeFileContents (pkgName : String) := s!"# {pkgName}"
+
 def mathToolchainUrl : String :=
   "https://raw.githubusercontent.com/leanprover-community/mathlib4/master/lean-toolchain"
 
@@ -267,6 +269,11 @@ def initPkg (dir : FilePath) (name : String) (tmp : InitTemplate) (lang : Config
     else
       logWarning "failed to initialize git repository"
 
+  -- Initialize a README.md file if none exists.
+  let readmeFile := dir / "README.md"
+  unless (← readmeFile.pathExists) do
+    IO.FS.writeFile readmeFile (readmeFileContents name)
+
 def validatePkgName (pkgName : String) : LogIO PUnit := do
   if pkgName.isEmpty || pkgName.all (· == '.') || pkgName.any (· ∈ ['/', '\\']) then
     error s!"illegal package name '{pkgName}'"

src/lake/Lake/Util/Log.lean

@@ -242,6 +242,7 @@ instance : FromJson Log := ⟨(Log.mk <$> fromJson? ·)⟩
 /-- A position in a `Log` (i.e., an array index). Can be past the log's end. -/
 structure Log.Pos where
   val : Nat
+  deriving Inhabited
 
 instance : OfNat Log.Pos (nat_lit 0) := ⟨⟨0⟩⟩
 

src/lake/Lake/Util/Newline.lean

@@ -1,24 +0,0 @@
-/-
-Copyright (c) 2023 Mario Carneiro. All rights reserved.
-Released under Apache 2.0 license as described in the file LICENSE.
-Authors: Mario Carneiro
--/
-
-namespace Lake
-
-/-- This is the same as `String.replace text "\r\n" "\n"`, but more efficient. -/
-@[inline] partial def crlf2lf (text : String) : String :=
-  go "" 0 0
-where
-  go (acc : String) (accStop pos : String.Pos) : String :=
-    if h : text.atEnd pos then
-      -- note: if accStop = 0 then acc is empty
-      if accStop = 0 then text else acc ++ text.extract accStop pos
-    else
-      let c := text.get' pos h
-      let pos' := text.next' pos h
-      if c == '\r' && text.get pos' == '\n' then
-        let acc := acc ++ text.extract accStop pos
-        go acc pos' (text.next pos')
-      else
-        go acc accStop pos'

src/lake/Lake/Util/Opaque.lean

@@ -34,7 +34,6 @@ macro (name := hydrateOpaqueType)
   let unsafeMk := mkIdent `unsafeMk
   let get := mkIdent `get
   let unsafeGet := mkIdent `unsafeGet
-  let get_mk := mkIdent `get_mk
   `(
   namespace $oty
   unsafe def $unsafeMk : $ty $args* → $oty $args* := unsafeCast
@@ -46,7 +45,5 @@ macro (name := hydrateOpaqueType)
   instance : Coe ($oty $args*) ($ty $args*) := ⟨$get⟩
 
   instance [Inhabited ($ty $args*)] : Inhabited ($oty $args*) := ⟨$mk default⟩
-
-  @[simp] axiom $get_mk $[{$args}]* {x : $ty $args*} : $get ($mk x) = x
   end $oty
   )

src/lake/Lake/Util/Task.lean

@@ -15,6 +15,7 @@ abbrev OptionTask := OptionT Task
 
 def BaseIOTask := Task
 instance : Monad BaseIOTask := inferInstanceAs <| Monad Task
+instance [Inhabited α] : Inhabited (BaseIOTask α) := inferInstance
 
 abbrev EIOTask ε := ExceptT ε BaseIOTask
 abbrev OptionIOTask := OptionT BaseIOTask

src/lake/tests/noRelease/clean.sh

@@ -1 +1 @@
-rm -rf .lake lake-manifest.json
+rm -rf .lake lake-manifest.json dep/.lake dep/.git

src/lake/tests/noRelease/dep/lakefile.lean

@@ -4,3 +4,4 @@ open Lake DSL
 package dep where
   preferReleaseBuild := true
   releaseRepo := "https://example.com"
+  buildArchive := "release.tgz"