chore: remove comment from wrong stdlib_flags.h

This PR removes a comment from wrong `stdlib_flags.h`. Only the one in `stage0/` should be edited.

.gitattributes

@@ -4,6 +4,7 @@ RELEASES.md merge=union
 stage0/** binary linguist-generated
 # The following file is often manually edited, so do show it in diffs
 stage0/src/stdlib_flags.h -binary -linguist-generated
+doc/std/grove/GroveStdlib/Generated/** linguist-generated
 # These files should not have line endings translated on Windows, because
 # it throws off parser tests. Later lines override earlier ones, so the
 # runner code is still treated as ordinary text.

.github/ISSUE_TEMPLATE/bug_report.md

@@ -9,7 +9,7 @@ assignees: ''
 
 ### Prerequisites
 
-Please put an X between the brackets as you perform the following steps:
+<!-- Please put an X between the brackets as you perform the following steps: -->
 
 * [ ] Check that your issue is not already filed:
       https://github.com/leanprover/lean4/issues

.github/workflows/ci.yml

@@ -262,12 +262,17 @@ jobs:
                 // Always run on large if available, more reliable regarding timeouts
                 "os": large ? "nscloud-ubuntu-22.04-amd64-8x16-with-cache" : "ubuntu-latest",
                 "enabled": level >= 2,
+                // do not fail nightlies on this for now
+                "secondary": level <= 2,
                 "test": true,
                 // turn off custom allocator & symbolic functions to make LSAN do its magic
                 "CMAKE_PRESET": "sanitize",
                 // `StackOverflow*` correctly triggers ubsan
                 // `reverse-ffi` fails to link in sanitizers
-                "CTEST_OPTIONS": "-E 'StackOverflow|reverse-ffi'"
+                // `interactive` and `async_select_channel` fail nondeterministically, would need to
+                // be investigated.
+                // 9366 is too close to timeout
+                "CTEST_OPTIONS": "-E 'StackOverflow|reverse-ffi|interactive|async_select_channel|9366'"
               },
               {
                 "name": "macOS",

.github/workflows/grove.yml

@@ -51,7 +51,7 @@ jobs:
       - name: Fetch upstream invalidated facts
         if: ${{ steps.should-run.outputs.should-run == 'true' && steps.workflow-info.outputs.pullRequestNumber != '' }}
         id: fetch-upstream
-        uses: TwoFx/grove-action/fetch-upstream@v0.4
+        uses: TwoFx/grove-action/fetch-upstream@v0.5
         with:
           artifact-name: grove-invalidated-facts
           base-ref: master
@@ -65,6 +65,7 @@ jobs:
           workflow: ci.yml
           path: artifacts
           name: "build-Linux release"
+          allow_forks: true
           name_is_regexp: true
 
       - name: Unpack toolchain
@@ -95,7 +96,7 @@ jobs:
       - name: Build
         if: ${{ steps.should-run.outputs.should-run == 'true' }}
         id: build
-        uses: TwoFx/grove-action/build@v0.4
+        uses: TwoFx/grove-action/build@v0.5
         with:
           project-path: doc/std/grove
           script-name: grove-stdlib

.github/workflows/pr-release.yml

@@ -127,7 +127,7 @@ jobs:
               description: "${{ github.repository_owner }}/lean4-pr-releases:pr-release-${{ steps.workflow-info.outputs.pullRequestNumber }}-${{ env.SHORT_SHA }}",
             });
 
-      - name: Add label
+      - name: Add toolchain-available label
         if: ${{ steps.workflow-info.outputs.pullRequestNumber != '' }}
         uses: actions/github-script@v8
         with:
@@ -515,6 +515,18 @@ jobs:
         run: |
           git push origin lean-pr-testing-${{ steps.workflow-info.outputs.pullRequestNumber }}
 
+      - name: Add mathlib4-nightly-available label
+        if: steps.workflow-info.outputs.pullRequestNumber != '' && steps.ready.outputs.mathlib_ready == 'true'
+        uses: actions/github-script@v8
+        with:
+          script: |
+            await github.rest.issues.addLabels({
+              issue_number: ${{ steps.workflow-info.outputs.pullRequestNumber }},
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              labels: ['mathlib4-nightly-available']
+            })
+
       # We next automatically create a reference manual branch using this toolchain.
       # Reference manual CI will be responsible for reporting back success or failure
       # to the PR comments asynchronously (and thus transitively SubVerso/Verso).

CMakeLists.txt

@@ -44,7 +44,9 @@ if (NOT ${CMAKE_SYSTEM_NAME} MATCHES "Emscripten")
     set(CADICAL_CXX c++)
     if (CADICAL_USE_CUSTOM_CXX)
       set(CADICAL_CXX ${CMAKE_CXX_COMPILER})
-      set(CADICAL_CXXFLAGS "${LEAN_EXTRA_CXX_FLAGS}")
+      # Use same platform flags as for Lean executables, in particular from `prepare-llvm-linux.sh`,
+      # but not Lean-specific `LEAN_EXTRA_CXX_FLAGS` such as fsanitize.
+      set(CADICAL_CXXFLAGS "${CMAKE_CXX_FLAGS}")
       set(CADICAL_LDFLAGS "-Wl,-rpath=\\$$ORIGIN/../lib")
     endif()
     find_program(CCACHE ccache)

doc/dev/bootstrap.md

@@ -72,6 +72,9 @@ update the archived C source code of the stage 0 compiler in `stage0/src`.
 
 The github repository will automatically update stage0 on `master` once
 `src/stdlib_flags.h` and `stage0/src/stdlib_flags.h` are out of sync.
+To trigger this, modify `stage0/src/stdlib_flags.h` (e.g., by adding or changing
+a comment). When `update-stage0` runs, it will overwrite `stage0/src/stdlib_flags.h`
+with the contents of `src/stdlib_flags.h`, bringing them back in sync.
 
 NOTE: A full rebuild of stage 1 will only be triggered when the *committed* contents of `stage0/` are changed.
 Thus if you change files in it manually instead of through `update-stage0-commit` (see below) or fetching updates from git, you either need to commit those changes first or run `make -C build/release clean-stdlib`.

doc/std/grove/GroveStdlib/Generated.lean

@@ -4,6 +4,7 @@ import GroveStdlib.Generated.«associative-creation-operations»
 import GroveStdlib.Generated.«associative-modification-operations»
 import GroveStdlib.Generated.«associative-create-then-query»
 import GroveStdlib.Generated.«associative-all-operations-covered»
+import GroveStdlib.Generated.«slice-producing»
 
 /-
 This file is autogenerated by grove. You can manually edit it, for example to resolve merge
@@ -20,3 +21,4 @@ def restoreState : RestoreStateM Unit := do
   «associative-modification-operations».restoreState
   «associative-create-then-query».restoreState
   «associative-all-operations-covered».restoreState
+  «slice-producing».restoreState

doc/std/grove/GroveStdlib/Generated/slice-producing.lean

@@ -0,0 +1,459 @@
+import Grove.Framework
+
+/-
+This file is autogenerated by grove. You can manually edit it, for example to resolve merge
+conflicts, but be careful.
+-/
+
+open Grove.Framework Widget
+
+namespace GroveStdlib.Generated.«slice-producing»
+
+def «c8a13d6d-7ed6-4cd1-a386-23e2d55ce6f7» : AssociationTable.Fact .declaration where
+  widgetId := "slice-producing"
+  factId := "c8a13d6d-7ed6-4cd1-a386-23e2d55ce6f7"
+  rowId := "c8a13d6d-7ed6-4cd1-a386-23e2d55ce6f7"
+  rowState := #[⟨"String", "String.slice", Declaration.def {
+    name := `String.slice
+    renderedStatement := "String.slice (s : String) (startInclusive endExclusive : s.Pos)\n  (h : startInclusive ≤ endExclusive) : String.Slice"
+    isDeprecated := false
+  }
+⟩,⟨"String.Slice", "String.Slice.slice", Declaration.def {
+    name := `String.Slice.slice
+    renderedStatement := "String.Slice.slice (s : String.Slice) (newStart newEnd : s.Pos) (h : newStart ≤ newEnd) :\n  String.Slice"
+    isDeprecated := false
+  }
+⟩,⟨"string-pos-forwards", "String.Pos.slice", Declaration.def {
+    name := `String.Pos.slice
+    renderedStatement := "String.Pos.slice {s : String} (pos p₀ p₁ : s.Pos) (h₁ : p₀ ≤ pos) (h₂ : pos ≤ p₁) :\n  (s.slice p₀ p₁ ⋯).Pos"
+    isDeprecated := false
+  }
+⟩,⟨"string-pos-backwards", "String.Pos.ofSlice", Declaration.def {
+    name := `String.Pos.ofSlice
+    renderedStatement := "String.Pos.ofSlice {s : String} {p₀ p₁ : s.Pos} {h : p₀ ≤ p₁} (pos : (s.slice p₀ p₁ h).Pos) : s.Pos"
+    isDeprecated := false
+  }
+⟩,⟨"string-slice-pos-forwards", "String.Slice.Pos.slice", Declaration.def {
+    name := `String.Slice.Pos.slice
+    renderedStatement := "String.Slice.Pos.slice {s : String.Slice} (pos p₀ p₁ : s.Pos) (h₁ : p₀ ≤ pos) (h₂ : pos ≤ p₁) :\n  (s.slice p₀ p₁ ⋯).Pos"
+    isDeprecated := false
+  }
+⟩,⟨"string-slice-pos-backwards", "String.Slice.Pos.ofSlice", Declaration.def {
+    name := `String.Slice.Pos.ofSlice
+    renderedStatement := "String.Slice.Pos.ofSlice {s : String.Slice} {p₀ p₁ : s.Pos} {h : p₀ ≤ p₁}\n  (pos : (s.slice p₀ p₁ h).Pos) : s.Pos"
+    isDeprecated := false
+  }
+⟩,⟨"string-pos-noproof", "String.Pos.sliceOrPanic", Declaration.def {
+    name := `String.Pos.sliceOrPanic
+    renderedStatement := "String.Pos.sliceOrPanic {s : String} (pos p₀ p₁ : s.Pos) {h : p₀ ≤ p₁} : (s.slice p₀ p₁ h).Pos"
+    isDeprecated := false
+  }
+⟩,⟨"string-slice-pos-noproof", "String.Slice.Pos.sliceOrPanic", Declaration.def {
+    name := `String.Slice.Pos.sliceOrPanic
+    renderedStatement := "String.Slice.Pos.sliceOrPanic {s : String.Slice} (pos p₀ p₁ : s.Pos) {h : p₀ ≤ p₁} :\n  (s.slice p₀ p₁ h).Pos"
+    isDeprecated := false
+  }
+⟩,]
+  metadata := {
+    status := .done
+    comment := ""
+  }
+def «21b4fdfd-f8b3-44f5-a59e-57f1dc1d6819» : AssociationTable.Fact .declaration where
+  widgetId := "slice-producing"
+  factId := "21b4fdfd-f8b3-44f5-a59e-57f1dc1d6819"
+  rowId := "21b4fdfd-f8b3-44f5-a59e-57f1dc1d6819"
+  rowState := #[⟨"String", "String.slice?", Declaration.def {
+    name := `String.slice?
+    renderedStatement := "String.slice? (s : String) (startInclusive endExclusive : s.Pos) : Option String.Slice"
+    isDeprecated := false
+  }
+⟩,⟨"String.Slice", "String.Slice.slice?", Declaration.def {
+    name := `String.Slice.slice?
+    renderedStatement := "String.Slice.slice? (s : String.Slice) (newStart newEnd : s.Pos) : Option String.Slice"
+    isDeprecated := false
+  }
+⟩,]
+  metadata := {
+    status := .postponed
+    comment := "Would be good to have better support"
+  }
+def «6f2b6ecb-2f0c-4e45-9da3-eb7f2e15eff0» : AssociationTable.Fact .declaration where
+  widgetId := "slice-producing"
+  factId := "6f2b6ecb-2f0c-4e45-9da3-eb7f2e15eff0"
+  rowId := "6f2b6ecb-2f0c-4e45-9da3-eb7f2e15eff0"
+  rowState := #[⟨"String", "String.slice!", Declaration.def {
+    name := `String.slice!
+    renderedStatement := "String.slice! (s : String) (p₁ p₂ : s.Pos) : String.Slice"
+    isDeprecated := false
+  }
+⟩,⟨"String.Slice", "String.Slice.slice!", Declaration.def {
+    name := `String.Slice.slice!
+    renderedStatement := "String.Slice.slice! (s : String.Slice) (newStart newEnd : s.Pos) : String.Slice"
+    isDeprecated := false
+  }
+⟩,⟨"string-pos-forwards", "String.Pos.slice!", Declaration.def {
+    name := `String.Pos.slice!
+    renderedStatement := "String.Pos.slice! {s : String} (pos p₀ p₁ : s.Pos) : (s.slice! p₀ p₁).Pos"
+    isDeprecated := false
+  }
+⟩,⟨"string-pos-backwards", "String.Pos.ofSlice!", Declaration.def {
+    name := `String.Pos.ofSlice!
+    renderedStatement := "String.Pos.ofSlice! {s : String} {p₀ p₁ : s.Pos} (pos : (s.slice! p₀ p₁).Pos) : s.Pos"
+    isDeprecated := false
+  }
+⟩,⟨"string-slice-pos-forwards", "String.Slice.Pos.slice!", Declaration.def {
+    name := `String.Slice.Pos.slice!
+    renderedStatement := "String.Slice.Pos.slice! {s : String.Slice} (pos p₀ p₁ : s.Pos) : (s.slice! p₀ p₁).Pos"
+    isDeprecated := false
+  }
+⟩,⟨"string-slice-pos-backwards", "String.Slice.Pos.ofSlice!", Declaration.def {
+    name := `String.Slice.Pos.ofSlice!
+    renderedStatement := "String.Slice.Pos.ofSlice! {s : String.Slice} {p₀ p₁ : s.Pos} (pos : (s.slice! p₀ p₁).Pos) : s.Pos"
+    isDeprecated := false
+  }
+⟩,]
+  metadata := {
+    status := .done
+    comment := ""
+  }
+def «a3bdf66d-bc11-4019-aee9-2f1c1701de52» : AssociationTable.Fact .declaration where
+  widgetId := "slice-producing"
+  factId := "a3bdf66d-bc11-4019-aee9-2f1c1701de52"
+  rowId := "a3bdf66d-bc11-4019-aee9-2f1c1701de52"
+  rowState := #[⟨"String", "String.trimAsciiStart", Declaration.def {
+    name := `String.trimAsciiStart
+    renderedStatement := "String.trimAsciiStart (s : String) : String.Slice"
+    isDeprecated := false
+  }
+⟩,⟨"String.Slice", "String.Slice.trimAsciiStart", Declaration.def {
+    name := `String.Slice.trimAsciiStart
+    renderedStatement := "String.Slice.trimAsciiStart (s : String.Slice) : String.Slice"
+    isDeprecated := false
+  }
+⟩,]
+  metadata := {
+    status := .bad
+    comment := "Missing `of` version at least"
+  }
+def «f12b2730-7a4d-465c-8a6d-9d051c300fd5» : AssociationTable.Fact .declaration where
+  widgetId := "slice-producing"
+  factId := "f12b2730-7a4d-465c-8a6d-9d051c300fd5"
+  rowId := "f12b2730-7a4d-465c-8a6d-9d051c300fd5"
+  rowState := #[⟨"String", "String.trimAsciiEnd", Declaration.def {
+    name := `String.trimAsciiEnd
+    renderedStatement := "String.trimAsciiEnd (s : String) : String.Slice"
+    isDeprecated := false
+  }
+⟩,⟨"String.Slice", "String.Slice.trimAsciiEnd", Declaration.def {
+    name := `String.Slice.trimAsciiEnd
+    renderedStatement := "String.Slice.trimAsciiEnd (s : String.Slice) : String.Slice"
+    isDeprecated := false
+  }
+⟩,]
+  metadata := {
+    status := .bad
+    comment := "Missing `of` version at least"
+  }
+def «32307b55-d6d1-4756-a947-dbe4dfde573c» : AssociationTable.Fact .declaration where
+  widgetId := "slice-producing"
+  factId := "32307b55-d6d1-4756-a947-dbe4dfde573c"
+  rowId := "32307b55-d6d1-4756-a947-dbe4dfde573c"
+  rowState := #[⟨"String", "String.trimAscii", Declaration.def {
+    name := `String.trimAscii
+    renderedStatement := "String.trimAscii (s : String) : String.Slice"
+    isDeprecated := false
+  }
+⟩,⟨"String.Slice", "String.Slice.trimAscii", Declaration.def {
+    name := `String.Slice.trimAscii
+    renderedStatement := "String.Slice.trimAscii (s : String.Slice) : String.Slice"
+    isDeprecated := false
+  }
+⟩,]
+  metadata := {
+    status := .bad
+    comment := "Missing `of` version at least\n"
+  }
+def «dce95a38-f55a-4d6a-ae79-078ffe4b5c15» : AssociationTable.Fact .declaration where
+  widgetId := "slice-producing"
+  factId := "dce95a38-f55a-4d6a-ae79-078ffe4b5c15"
+  rowId := "dce95a38-f55a-4d6a-ae79-078ffe4b5c15"
+  rowState := #[⟨"String", "String.toSlice", Declaration.def {
+    name := `String.toSlice
+    renderedStatement := "String.toSlice (s : String) : String.Slice"
+    isDeprecated := false
+  }
+⟩,⟨"string-pos-forwards", "String.Pos.toSlice", Declaration.def {
+    name := `String.Pos.toSlice
+    renderedStatement := "String.Pos.toSlice {s : String} (pos : s.Pos) : s.toSlice.Pos"
+    isDeprecated := false
+  }
+⟩,⟨"string-pos-backwards", "String.Pos.ofToSlice", Declaration.def {
+    name := `String.Pos.ofToSlice
+    renderedStatement := "String.Pos.ofToSlice {s : String} (pos : s.toSlice.Pos) : s.Pos"
+    isDeprecated := false
+  }
+⟩,]
+  metadata := {
+    status := .done
+    comment := ""
+  }
+def «005a3f30-5dab-493f-b168-32c36a2bdf7c» : AssociationTable.Fact .declaration where
+  widgetId := "slice-producing"
+  factId := "005a3f30-5dab-493f-b168-32c36a2bdf7c"
+  rowId := "005a3f30-5dab-493f-b168-32c36a2bdf7c"
+  rowState := #[⟨"String.Slice", "String.Slice.str", Declaration.def {
+    name := `String.Slice.str
+    renderedStatement := "String.Slice.str (self : String.Slice) : String"
+    isDeprecated := false
+  }
+⟩,⟨"string-slice-pos-forwards", "String.Slice.Pos.str", Declaration.def {
+    name := `String.Slice.Pos.str
+    renderedStatement := "String.Slice.Pos.str {s : String.Slice} (pos : s.Pos) : s.str.Pos"
+    isDeprecated := false
+  }
+⟩,⟨"string-slice-pos-backwards", "String.Slice.Pos.ofStr", Declaration.def {
+    name := `String.Slice.Pos.ofStr
+    renderedStatement := "String.Slice.Pos.ofStr {s : String.Slice} (pos : s.str.Pos) (h₁ : s.startInclusive ≤ pos)\n  (h₂ : pos ≤ s.endExclusive) : s.Pos"
+    isDeprecated := false
+  }
+⟩,]
+  metadata := {
+    status := .bad
+    comment := "Missing `no proof` version\n"
+  }
+def «5f1a154c-ae2f-43a1-9409-2ce95b163ef3» : AssociationTable.Fact .declaration where
+  widgetId := "slice-producing"
+  factId := "5f1a154c-ae2f-43a1-9409-2ce95b163ef3"
+  rowId := "5f1a154c-ae2f-43a1-9409-2ce95b163ef3"
+  rowState := #[⟨"String", "String.drop", Declaration.def {
+    name := `String.drop
+    renderedStatement := "String.drop (s : String) (n : Nat) : String.Slice"
+    isDeprecated := false
+  }
+⟩,⟨"String.Slice", "String.Slice.drop", Declaration.def {
+    name := `String.Slice.drop
+    renderedStatement := "String.Slice.drop (s : String.Slice) (n : Nat) : String.Slice"
+    isDeprecated := false
+  }
+⟩,]
+  metadata := {
+    status := .bad
+    comment := "Missing position transformations"
+  }
+def «179518d1-ad07-4b2b-8ffe-3b7616e4c4ab» : AssociationTable.Fact .declaration where
+  widgetId := "slice-producing"
+  factId := "179518d1-ad07-4b2b-8ffe-3b7616e4c4ab"
+  rowId := "179518d1-ad07-4b2b-8ffe-3b7616e4c4ab"
+  rowState := #[⟨"String", "String.take", Declaration.def {
+    name := `String.take
+    renderedStatement := "String.take (s : String) (n : Nat) : String.Slice"
+    isDeprecated := false
+  }
+⟩,⟨"String.Slice", "String.Slice.take", Declaration.def {
+    name := `String.Slice.take
+    renderedStatement := "String.Slice.take (s : String.Slice) (n : Nat) : String.Slice"
+    isDeprecated := false
+  }
+⟩,]
+  metadata := {
+    status := .bad
+    comment := "Missing position transformations"
+  }
+def «55c587fd-a7a8-4633-a4ae-e2c4e768ad28» : AssociationTable.Fact .declaration where
+  widgetId := "slice-producing"
+  factId := "55c587fd-a7a8-4633-a4ae-e2c4e768ad28"
+  rowId := "55c587fd-a7a8-4633-a4ae-e2c4e768ad28"
+  rowState := #[⟨"String", "String.dropWhile", Declaration.def {
+    name := `String.dropWhile
+    renderedStatement := "String.dropWhile {ρ : Type} (s : String) (pat : ρ) [String.Slice.Pattern.ForwardPattern pat] :\n  String.Slice"
+    isDeprecated := false
+  }
+⟩,⟨"String.Slice", "String.Slice.dropWhile", Declaration.def {
+    name := `String.Slice.dropWhile
+    renderedStatement := "String.Slice.dropWhile {ρ : Type} (s : String.Slice) (pat : ρ)\n  [String.Slice.Pattern.ForwardPattern pat] : String.Slice"
+    isDeprecated := false
+  }
+⟩,]
+  metadata := {
+    status := .bad
+    comment := "Missing position transformations"
+  }
+def «d4444684-4279-4400-9be2-561a7cdb32c1» : AssociationTable.Fact .declaration where
+  widgetId := "slice-producing"
+  factId := "d4444684-4279-4400-9be2-561a7cdb32c1"
+  rowId := "d4444684-4279-4400-9be2-561a7cdb32c1"
+  rowState := #[⟨"String", "String.takeWhile", Declaration.def {
+    name := `String.takeWhile
+    renderedStatement := "String.takeWhile {ρ : Type} (s : String) (pat : ρ) [String.Slice.Pattern.ForwardPattern pat] :\n  String.Slice"
+    isDeprecated := false
+  }
+⟩,⟨"String.Slice", "String.Slice.takeWhile", Declaration.def {
+    name := `String.Slice.takeWhile
+    renderedStatement := "String.Slice.takeWhile {ρ : Type} (s : String.Slice) (pat : ρ)\n  [String.Slice.Pattern.ForwardPattern pat] : String.Slice"
+    isDeprecated := false
+  }
+⟩,]
+  metadata := {
+    status := .bad
+    comment := "Missing position transformations"
+  }
+def «1c9e6689-65a0-4d4b-b001-256e83917d98» : AssociationTable.Fact .declaration where
+  widgetId := "slice-producing"
+  factId := "1c9e6689-65a0-4d4b-b001-256e83917d98"
+  rowId := "1c9e6689-65a0-4d4b-b001-256e83917d98"
+  rowState := #[⟨"String", "String.dropEndWhile", Declaration.def {
+    name := `String.dropEndWhile
+    renderedStatement := "String.dropEndWhile {ρ : Type} (s : String) (pat : ρ) [String.Slice.Pattern.BackwardPattern pat] :\n  String.Slice"
+    isDeprecated := false
+  }
+⟩,⟨"String.Slice", "String.Slice.dropEndWhile", Declaration.def {
+    name := `String.Slice.dropEndWhile
+    renderedStatement := "String.Slice.dropEndWhile {ρ : Type} (s : String.Slice) (pat : ρ)\n  [String.Slice.Pattern.BackwardPattern pat] : String.Slice"
+    isDeprecated := false
+  }
+⟩,]
+  metadata := {
+    status := .bad
+    comment := "Missing position transformations"
+  }
+def «b836052b-3470-4a8e-8989-6951c898de37» : AssociationTable.Fact .declaration where
+  widgetId := "slice-producing"
+  factId := "b836052b-3470-4a8e-8989-6951c898de37"
+  rowId := "b836052b-3470-4a8e-8989-6951c898de37"
+  rowState := #[⟨"String", "String.takeEndWhile", Declaration.def {
+    name := `String.takeEndWhile
+    renderedStatement := "String.takeEndWhile {ρ : Type} (s : String) (pat : ρ) [String.Slice.Pattern.BackwardPattern pat] :\n  String.Slice"
+    isDeprecated := false
+  }
+⟩,⟨"String.Slice", "String.Slice.takeEndWhile", Declaration.def {
+    name := `String.Slice.takeEndWhile
+    renderedStatement := "String.Slice.takeEndWhile {ρ : Type} (s : String.Slice) (pat : ρ)\n  [String.Slice.Pattern.BackwardPattern pat] : String.Slice"
+    isDeprecated := false
+  }
+⟩,]
+  metadata := {
+    status := .bad
+    comment := "Missing position transformations"
+  }
+def «5aa777d8-9642-43d8-9e20-30400fb8bb9d» : AssociationTable.Fact .declaration where
+  widgetId := "slice-producing"
+  factId := "5aa777d8-9642-43d8-9e20-30400fb8bb9d"
+  rowId := "5aa777d8-9642-43d8-9e20-30400fb8bb9d"
+  rowState := #[⟨"String", "String.dropPrefix", Declaration.def {
+    name := `String.dropPrefix
+    renderedStatement := "String.dropPrefix {ρ : Type} (s : String) (pat : ρ) [String.Slice.Pattern.ForwardPattern pat] :\n  String.Slice"
+    isDeprecated := false
+  }
+⟩,⟨"String.Slice", "String.Slice.dropPrefix", Declaration.def {
+    name := `String.Slice.dropPrefix
+    renderedStatement := "String.Slice.dropPrefix {ρ : Type} (s : String.Slice) (pat : ρ)\n  [String.Slice.Pattern.ForwardPattern pat] : String.Slice"
+    isDeprecated := false
+  }
+⟩,]
+  metadata := {
+    status := .bad
+    comment := "Missing position transformations"
+  }
+def «80e3869d-fcfe-459d-8433-fe221f7b3c7a» : AssociationTable.Fact .declaration where
+  widgetId := "slice-producing"
+  factId := "80e3869d-fcfe-459d-8433-fe221f7b3c7a"
+  rowId := "80e3869d-fcfe-459d-8433-fe221f7b3c7a"
+  rowState := #[⟨"String", "String.dropSuffix", Declaration.def {
+    name := `String.dropSuffix
+    renderedStatement := "String.dropSuffix {ρ : Type} (s : String) (pat : ρ) [String.Slice.Pattern.BackwardPattern pat] :\n  String.Slice"
+    isDeprecated := false
+  }
+⟩,⟨"String.Slice", "String.Slice.dropSuffix", Declaration.def {
+    name := `String.Slice.dropSuffix
+    renderedStatement := "String.Slice.dropSuffix {ρ : Type} (s : String.Slice) (pat : ρ)\n  [String.Slice.Pattern.BackwardPattern pat] : String.Slice"
+    isDeprecated := false
+  }
+⟩,]
+  metadata := {
+    status := .bad
+    comment := "Missing position transformations"
+  }
+def «4feda3e0-903b-4d52-b34e-0af70f7866e0» : AssociationTable.Fact .declaration where
+  widgetId := "slice-producing"
+  factId := "4feda3e0-903b-4d52-b34e-0af70f7866e0"
+  rowId := "4feda3e0-903b-4d52-b34e-0af70f7866e0"
+  rowState := #[⟨"String", "String.dropPrefix?", Declaration.def {
+    name := `String.dropPrefix?
+    renderedStatement := "String.dropPrefix? {ρ : Type} (s : String) (pat : ρ) [String.Slice.Pattern.ForwardPattern pat] :\n  Option String.Slice"
+    isDeprecated := false
+  }
+⟩,⟨"String.Slice", "String.Slice.dropPrefix?", Declaration.def {
+    name := `String.Slice.dropPrefix?
+    renderedStatement := "String.Slice.dropPrefix? {ρ : Type} (s : String.Slice) (pat : ρ)\n  [String.Slice.Pattern.ForwardPattern pat] : Option String.Slice"
+    isDeprecated := false
+  }
+⟩,]
+  metadata := {
+    status := .postponed
+    comment := "Missing position transformations"
+  }
+def «45ca44c8-fbd5-4400-8297-a60778f302b0» : AssociationTable.Fact .declaration where
+  widgetId := "slice-producing"
+  factId := "45ca44c8-fbd5-4400-8297-a60778f302b0"
+  rowId := "45ca44c8-fbd5-4400-8297-a60778f302b0"
+  rowState := #[⟨"String", "String.dropSuffix?", Declaration.def {
+    name := `String.dropSuffix?
+    renderedStatement := "String.dropSuffix? {ρ : Type} (s : String) (pat : ρ) [String.Slice.Pattern.BackwardPattern pat] :\n  Option String.Slice"
+    isDeprecated := false
+  }
+⟩,⟨"String.Slice", "String.Slice.dropSuffix?", Declaration.def {
+    name := `String.Slice.dropSuffix?
+    renderedStatement := "String.Slice.dropSuffix? {ρ : Type} (s : String.Slice) (pat : ρ)\n  [String.Slice.Pattern.BackwardPattern pat] : Option String.Slice"
+    isDeprecated := false
+  }
+⟩,]
+  metadata := {
+    status := .postponed
+    comment := "Missing position transformations"
+  }
+
+def table : AssociationTable.Data .declaration where
+  widgetId := "slice-producing"
+  rows := #[
+    ⟨"c8a13d6d-7ed6-4cd1-a386-23e2d55ce6f7", "slice", #[⟨"String", "String.slice"⟩,⟨"String.Slice", "String.Slice.slice"⟩,⟨"string-pos-forwards", "String.Pos.slice"⟩,⟨"string-pos-backwards", "String.Pos.ofSlice"⟩,⟨"string-slice-pos-forwards", "String.Slice.Pos.slice"⟩,⟨"string-slice-pos-backwards", "String.Slice.Pos.ofSlice"⟩,⟨"string-pos-noproof", "String.Pos.sliceOrPanic"⟩,⟨"string-slice-pos-noproof", "String.Slice.Pos.sliceOrPanic"⟩,]⟩,
+    ⟨"21b4fdfd-f8b3-44f5-a59e-57f1dc1d6819", "slice?", #[⟨"String", "String.slice?"⟩,⟨"String.Slice", "String.Slice.slice?"⟩,]⟩,
+    ⟨"6f2b6ecb-2f0c-4e45-9da3-eb7f2e15eff0", "slice!", #[⟨"String", "String.slice!"⟩,⟨"String.Slice", "String.Slice.slice!"⟩,⟨"string-pos-forwards", "String.Pos.slice!"⟩,⟨"string-pos-backwards", "String.Pos.ofSlice!"⟩,⟨"string-slice-pos-forwards", "String.Slice.Pos.slice!"⟩,⟨"string-slice-pos-backwards", "String.Slice.Pos.ofSlice!"⟩,]⟩,
+    ⟨"a3bdf66d-bc11-4019-aee9-2f1c1701de52", "trimAsciiStart", #[⟨"String", "String.trimAsciiStart"⟩,⟨"String.Slice", "String.Slice.trimAsciiStart"⟩,]⟩,
+    ⟨"f12b2730-7a4d-465c-8a6d-9d051c300fd5", "trimAsciiEnd", #[⟨"String", "String.trimAsciiEnd"⟩,⟨"String.Slice", "String.Slice.trimAsciiEnd"⟩,]⟩,
+    ⟨"32307b55-d6d1-4756-a947-dbe4dfde573c", "trimAscii", #[⟨"String", "String.trimAscii"⟩,⟨"String.Slice", "String.Slice.trimAscii"⟩,]⟩,
+    ⟨"dce95a38-f55a-4d6a-ae79-078ffe4b5c15", "toSlice", #[⟨"String", "String.toSlice"⟩,⟨"string-pos-forwards", "String.Pos.toSlice"⟩,⟨"string-pos-backwards", "String.Pos.ofToSlice"⟩,]⟩,
+    ⟨"005a3f30-5dab-493f-b168-32c36a2bdf7c", "str", #[⟨"String.Slice", "String.Slice.str"⟩,⟨"string-slice-pos-forwards", "String.Slice.Pos.str"⟩,⟨"string-slice-pos-backwards", "String.Slice.Pos.ofStr"⟩,]⟩,
+    ⟨"5f1a154c-ae2f-43a1-9409-2ce95b163ef3", "drop", #[⟨"String", "String.drop"⟩,⟨"String.Slice", "String.Slice.drop"⟩,]⟩,
+    ⟨"179518d1-ad07-4b2b-8ffe-3b7616e4c4ab", "take", #[⟨"String", "String.take"⟩,⟨"String.Slice", "String.Slice.take"⟩,]⟩,
+    ⟨"55c587fd-a7a8-4633-a4ae-e2c4e768ad28", "dropWhile", #[⟨"String", "String.dropWhile"⟩,⟨"String.Slice", "String.Slice.dropWhile"⟩,]⟩,
+    ⟨"d4444684-4279-4400-9be2-561a7cdb32c1", "takeWhile", #[⟨"String", "String.takeWhile"⟩,⟨"String.Slice", "String.Slice.takeWhile"⟩,]⟩,
+    ⟨"1c9e6689-65a0-4d4b-b001-256e83917d98", "dropEndWhile", #[⟨"String", "String.dropEndWhile"⟩,⟨"String.Slice", "String.Slice.dropEndWhile"⟩,]⟩,
+    ⟨"b836052b-3470-4a8e-8989-6951c898de37", "takeEndWhile", #[⟨"String", "String.takeEndWhile"⟩,⟨"String.Slice", "String.Slice.takeEndWhile"⟩,]⟩,
+    ⟨"5aa777d8-9642-43d8-9e20-30400fb8bb9d", "dropPrefix", #[⟨"String", "String.dropPrefix"⟩,⟨"String.Slice", "String.Slice.dropPrefix"⟩,]⟩,
+    ⟨"80e3869d-fcfe-459d-8433-fe221f7b3c7a", "dropSuffix", #[⟨"String", "String.dropSuffix"⟩,⟨"String.Slice", "String.Slice.dropSuffix"⟩,]⟩,
+    ⟨"4feda3e0-903b-4d52-b34e-0af70f7866e0", "dropPrefix?", #[⟨"String", "String.dropPrefix?"⟩,⟨"String.Slice", "String.Slice.dropPrefix?"⟩,]⟩,
+    ⟨"45ca44c8-fbd5-4400-8297-a60778f302b0", "dropSuffix?", #[⟨"String", "String.dropSuffix?"⟩,⟨"String.Slice", "String.Slice.dropSuffix?"⟩,]⟩,
+  ]
+  facts := #[
+    «c8a13d6d-7ed6-4cd1-a386-23e2d55ce6f7»,
+    «21b4fdfd-f8b3-44f5-a59e-57f1dc1d6819»,
+    «6f2b6ecb-2f0c-4e45-9da3-eb7f2e15eff0»,
+    «a3bdf66d-bc11-4019-aee9-2f1c1701de52»,
+    «f12b2730-7a4d-465c-8a6d-9d051c300fd5»,
+    «32307b55-d6d1-4756-a947-dbe4dfde573c»,
+    «dce95a38-f55a-4d6a-ae79-078ffe4b5c15»,
+    «005a3f30-5dab-493f-b168-32c36a2bdf7c»,
+    «5f1a154c-ae2f-43a1-9409-2ce95b163ef3»,
+    «179518d1-ad07-4b2b-8ffe-3b7616e4c4ab»,
+    «55c587fd-a7a8-4633-a4ae-e2c4e768ad28»,
+    «d4444684-4279-4400-9be2-561a7cdb32c1»,
+    «1c9e6689-65a0-4d4b-b001-256e83917d98»,
+    «b836052b-3470-4a8e-8989-6951c898de37»,
+    «5aa777d8-9642-43d8-9e20-30400fb8bb9d»,
+    «80e3869d-fcfe-459d-8433-fe221f7b3c7a»,
+    «4feda3e0-903b-4d52-b34e-0af70f7866e0»,
+    «45ca44c8-fbd5-4400-8297-a60778f302b0»,
+  ]
+
+def restoreState : RestoreStateM Unit := do
+  addAssociationTable table

doc/std/grove/GroveStdlib/Std.lean

@@ -15,7 +15,7 @@ namespace GroveStdlib
 namespace Std
 
 def introduction : Node :=
-  .text "Welcome to the interactive Lean standard library outline!"
+  .text ⟨"introduction", "Welcome to the interactive Lean standard library outline!"⟩
 
 end Std
 

doc/std/grove/GroveStdlib/Std/CoreTypesAndOperations/StringsAndFormatting.lean

@@ -11,9 +11,28 @@ namespace GroveStdlib.Std.CoreTypesAndOperations
 
 namespace StringsAndFormatting
 
+def highLevelStringTypes : List Lean.Name :=
+  [`String, `String.Slice, `String.Pos, `String.Slice.Pos]
+
+def sliceProducing : AssociationTable (β := Alias Lean.Name) .declaration
+    [`String, `String.Slice,
+     Alias.mk `String.Pos "string-pos-forwards" "String.Pos (forwards)",
+     Alias.mk `String.Pos "string-pos-backwards" "String.Pos (backwards)",
+     Alias.mk `String.Pos "string-pos-noproof" "String.Pos (no proof)",
+     Alias.mk `String.Slice.Pos "string-slice-pos-forwards" "String.Slice.Pos (forwards)",
+     Alias.mk `String.Slice.Pos "string-slice-pos-backwards" "String.Slice.Pos (backwards)",
+     Alias.mk `String.Slice.Pos "string-slice-pos-noproof" "String.Slice.Pos (no proof)"] where
+  id := "slice-producing"
+  title := "String functions returning slices"
+  description := "Operations on strings and string slices that themselves return a new string slice."
+  dataSources n := DataSource.definitionsInNamespace n.inner
+
 end StringsAndFormatting
 
-def stringsAndFormatting : Node :=
-  .section "strings-and-formatting" "Strings and formatting" #[]
+open StringsAndFormatting
 
-end GroveStdlib.Std.CoreTypesAndOperations
+def stringsAndFormatting : Node :=
+  .section "strings-and-formatting" "Strings and formatting"
+    #[.associationTable sliceProducing]
+
+end GroveStdlib.Std.CoreTypesAndOperations

doc/std/grove/lake-manifest.json

@@ -5,7 +5,7 @@
    "type": "git",
    "subDir": "backend",
    "scope": "",
-   "rev": "3e8aabdea58c11813c5d3b7eeb187ded44ee9a34",
+   "rev": "c19abde9101b29eb1e30d0d9d9e24ec2714f25b6",
    "name": "grove",
    "manifestFile": "lake-manifest.json",
    "inputRev": "master",
@@ -15,10 +15,10 @@
    "type": "git",
    "subDir": null,
    "scope": "leanprover",
-   "rev": "1604206fcd0462da9a241beeac0e2df471647435",
+   "rev": "d9fc8ae23024be37424a189982c92356e37935c8",
    "name": "Cli",
    "manifestFile": "lake-manifest.json",
-   "inputRev": "main",
+   "inputRev": "nightly-testing",
    "inherited": true,
    "configFile": "lakefile.toml"}],
  "name": "grovestdlib",

script/Shake.lean

@@ -3,16 +3,21 @@ Copyright (c) 2023 Mario Carneiro. All rights reserved.
 Released under Apache 2.0 license as described in the file LICENSE.
 Authors: Mario Carneiro, Sebastian Ullrich
 -/
-import Lake.CLI.Main
+module
+
+import Lean.Environment
 import Lean.ExtraModUses
 
-/-! # `lake exe shake` command
+import Lake.CLI.Main
+import Lean.Parser.Module
+import Lake.Load.Workspace
+
+/-! # Shake: A Lean import minimizer
 
 This command will check the current project (or a specified target module) and all dependencies for
 unused imports. This works by looking at generated `.olean` files to deduce required imports and
 ensuring that every import is used to contribute some constant or other elaboration dependency
-recorded by `recordExtraModUse`. Because recompilation is not needed this is quite fast (about 8
-seconds to check `Mathlib` and all dependencies).
+recorded by `recordExtraModUse` and friends.
 -/
 
 /-- help string for the command line interface -/
@@ -28,13 +33,83 @@ Options:
   --force
     Skips the `lake build --no-build` sanity check
 
+  --keep-implied
+    Preserves existing imports that are implied by other imports and thus not technically needed
+    anymore
+
+  --keep-prefix
+    If an import `X` would be replaced in favor of a more specific import `X.Y...` it implies,
+    preserves the original import instead. More generally, prefers inserting `import X` even if it
+    was not part of the original imports as long as it was in the original transitive import closure
+    of the current module.
+
+  --keep-public
+    Preserves all `public` imports to avoid breaking changes for external downstream modules
+
+  --add-public
+    Adds new imports as `public` if they have been in the original public closure of that module.
+    In other words, public imports will not be removed from a module unless they are unused even
+    in the private scope, and those that are removed will be re-added as `public` in downstream
+    modules even if only needed in the private scope there. Unlike `--keep-public`, this may
+    introduce breaking changes but will still limit the number of inserted imports.
+
+  --explain
+    Gives constants explaining why each module is needed
+
   --fix
     Apply the suggested fixes directly. Make sure you have a clean checkout
     before running this, so you can review the changes.
+
+  --gh-style
+    Outputs messages that can be parsed by `gh-problem-matcher-wrap`
+
+Annotations:
+  The following annotations can be added to Lean files in order to configure the behavior of
+  `shake`. Only the substring `shake: ` directly followed by a directive is checked for, so multiple
+  directives can be mixed in one line such as `-- shake: keep-downstream, shake: keep-all`, and they
+  can be surrounded by arbitrary comments such as `-- shake: keep (metaprogram output dependency)`.
+
+  * `module -- shake: keep-downstream`:
+    Preserves this module in all (current) downstream modules, adding new imports of it if needed.
+
+  * `module -- shake: keep-all`:
+    Preserves all existing imports in this module as is. New imports now needed because of upstream
+    changes may still be added.
+
+  * `import X -- shake: keep`:
+    Preserves this specific import in the current module. The most common use case is to preserve a
+    public import that will be needed in downstream modules to make sense of the output of a
+    metaprogram defined in this module. For example, if a tactic is defined that may synthesize a
+    reference to a theorem when run, there is no way for `shake` to detect this by itself and the
+    module of that theorem should be publicly imported and annotated with `keep` in the tactic's
+    module.
+    ```
+    public import X  -- shake: keep (metaprogram output dependency)
+
+    ...
+
+    elab \"my_tactic\" : tactic => do
+      ... mkConst ``f -- `f`, defined in `X`, may appear in the output of this tactic
+    ```
 "
 
 open Lean
 
+/-- The parsed CLI arguments. See `help` for more information -/
+structure Args where
+  help : Bool := false
+  keepImplied : Bool := false
+  keepPrefix : Bool := false
+  keepPublic : Bool := false
+  addPublic : Bool := false
+  force : Bool := false
+  githubStyle : Bool := false
+  explain : Bool := false
+  trace : Bool := false
+  fix : Bool := false
+  /-- `<MODULE>..`: the list of root modules to check -/
+  mods : Array Name := #[]
+
 /-- We use `Nat` as a bitset for doing efficient set operations.
 The bit indexes will usually be a module index. -/
 structure Bitset where
@@ -88,7 +163,7 @@ def ofImport : Lean.Import → NeedsKind
 
 end NeedsKind
 
-/-- Logically, a map `NeedsKind → Bitset`. -/
+/-- Logically, a map `NeedsKind → Set ModuleIdx`, or `Set Import`. -/
 structure Needs where
   pub : Bitset
   priv : Bitset
@@ -124,6 +199,20 @@ def Needs.union (needs : Needs) (k : NeedsKind) (s : Bitset) : Needs :=
 def Needs.sub (needs : Needs) (k : NeedsKind) (s : Bitset) : Needs :=
   needs.modify k (fun s' => s' ^^^ (s' ∩ s))
 
+instance : Union Needs where
+  union a b := {
+    pub := a.pub ∪ b.pub
+    priv := a.priv ∪ b.priv
+    metaPub := a.metaPub ∪ b.metaPub
+    metaPriv := a.metaPriv ∪ b.metaPriv }
+
+/-- The list of edits that will be applied in `--fix`. `edits[i] = (removed, added)` where:
+
+* If `j ∈ removed` then we want to delete module named `j` from the imports of `i`
+* If `j ∈ added` then we want to add module index `j` to the imports of `i`.
+-/
+abbrev Edits := Std.HashMap Name (Array Import × Array Import)
+
 /-- The main state of the checker, containing information on all loaded modules. -/
 structure State where
   env  : Environment
@@ -143,6 +232,10 @@ structure State where
   changes to upstream headers.
   -/
   transDepsOrig : Array Needs := #[]
+  /-- Modules that should always be preserved downstream. -/
+  preserve : Needs := default
+  /-- Edits to be applied to the module imports. -/
+  edits : Edits := {}
 
 def State.mods (s : State) := s.env.header.moduleData
 def State.modNames (s : State) := s.env.header.moduleNames
@@ -185,13 +278,36 @@ def addTransitiveImps (transImps : Needs) (imp : Import) (j : Nat) (impTransImps
 
   transImps
 
+def isDeclMeta' (env : Environment) (declName : Name) : Bool :=
+  -- Matchers are not compiled by themselves but inlined by the compiler, so there is no IR decl
+  -- to be tagged as `meta`.
+  -- TODO: It would be better to base the entire `meta` inference on the IR only and consider module
+  -- references from any other context as compatible with both phases.
+  let inferFor :=
+    if declName.isStr && (declName.getString!.startsWith "match_" || declName.getString! == "_unsafe_rec") then declName.getPrefix else declName
+  isDeclMeta env inferFor
+
+/--
+Given an `Expr` reference, returns the declaration name that should be considered the reference, if
+any.
+-/
+def getDepConstName? (env : Environment) (ref : Name) : Option Name := do
+  -- Ignore references to reserved names, they can be re-generated in-place
+  guard <| !isReservedName env ref
+  -- `_simp_...` constants are similar, use base decl instead
+  return if ref.isStr && ref.getString!.startsWith "_simp_" then
+    ref.getPrefix
+  else
+    ref
+
 /-- Calculates the needs for a given module `mod` from constants and recorded extra uses. -/
-def calcNeeds (env : Environment) (i : ModuleIdx) : Needs := Id.run do
+def calcNeeds (s : State) (i : ModuleIdx) : Needs := Id.run do
+  let env := s.env
   let mut needs := default
   for ci in env.header.moduleData[i]!.constants do
     -- Added guard for cases like `structure` that are still exported even if private
     let pubCI? := guard (!isPrivateName ci.name) *> (env.setExporting true).find? ci.name
-    let k := { isExported := pubCI?.isSome, isMeta := isMeta env ci.name }
+    let k := { isExported := pubCI?.isSome, isMeta := isDeclMeta' env ci.name }
     needs := visitExpr k ci.type needs
     if let some e := ci.value? (allowOpaque := true) then
       -- type and value has identical visibility under `meta`
@@ -206,12 +322,19 @@ def calcNeeds (env : Environment) (i : ModuleIdx) : Needs := Id.run do
   return needs
 where
   /-- Accumulate the results from expression `e` into `deps`. -/
-  visitExpr (k : NeedsKind) e deps :=
-    Lean.Expr.foldConsts e deps fun c deps => match env.getModuleIdxFor? c with
-      | some j =>
-        let k := { k with isMeta := k.isMeta && !isMeta env c }
-        if j != i then deps.union k {j} else deps
-      | _ => deps
+  visitExpr (k : NeedsKind) (e : Expr) (deps : Needs) : Needs :=
+    let env := s.env
+    Lean.Expr.foldConsts e deps fun c deps => Id.run do
+      let mut deps := deps
+      if let some c := getDepConstName? env c then
+        if let some j := env.getModuleIdxFor? c then
+          let k := { k with isMeta := k.isMeta && !isDeclMeta' env c }
+          if j != i then
+            deps := deps.union k {j}
+            for indMod in (indirectModUseExt.getState env)[c]?.getD #[] do
+              if s.transDeps[i]!.has k indMod then
+                deps := deps.union k {indMod}
+      return deps
 
 /--
 Calculates the same as `calcNeeds` but tracing each module to a use-def declaration pair or
@@ -223,7 +346,7 @@ def getExplanations (env : Environment) (i : ModuleIdx) :
   for ci in env.header.moduleData[i]!.constants do
     -- Added guard for cases like `structure` that are still exported even if private
     let pubCI? := guard (!isPrivateName ci.name) *> (env.setExporting true).find? ci.name
-    let k := { isExported := pubCI?.isSome, isMeta := isMeta env ci.name }
+    let k := { isExported := pubCI?.isSome, isMeta := isDeclMeta' env ci.name }
     deps := visitExpr k ci.name ci.type deps
     if let some e := ci.value? (allowOpaque := true) then
       let k := if k.isMeta then k else
@@ -239,18 +362,18 @@ def getExplanations (env : Environment) (i : ModuleIdx) :
 where
   /-- Accumulate the results from expression `e` into `deps`. -/
   visitExpr (k : NeedsKind) name e deps :=
-    Lean.Expr.foldConsts e deps fun c deps => match env.getModuleIdxFor? c with
-      | some i =>
-        let k := { k with isMeta := k.isMeta && !isMeta env c }
-        if
-          if let some (some (name', _)) := deps[(i, k)]? then
-            decide (name.toString.length < name'.toString.length)
-          else true
-        then
-          deps.insert (i, k) (name, c)
-        else
-          deps
-      | _ => deps
+    Lean.Expr.foldConsts e deps fun c deps => Id.run do
+      let mut deps := deps
+      if let some c := getDepConstName? env c then
+        if let some j := env.getModuleIdxFor? c then
+          let k := { k with isMeta := k.isMeta && !isDeclMeta' env c }
+          if
+            if let some (some (name', _)) := deps[(j, k)]? then
+              decide (name.toString.length < name'.toString.length)
+            else true
+          then
+            deps := deps.insert (j, k) (name, c)
+      return deps
 
 partial def initStateFromEnv (env : Environment) : State := Id.run do
   let mut s := { env }
@@ -266,13 +389,6 @@ partial def initStateFromEnv (env : Environment) : State := Id.run do
   s := { s with transDepsOrig := s.transDeps }
   return s
 
-/-- The list of edits that will be applied in `--fix`. `edits[i] = (removed, added)` where:
-
-* If `j ∈ removed` then we want to delete module named `j` from the imports of `i`
-* If `j ∈ added` then we want to add module index `j` to the imports of `i`.
--/
-abbrev Edits := Std.HashMap Name (Array Import × Array Import)
-
 /-- Register that we want to remove `tgt` from the imports of `src`. -/
 def Edits.remove (ed : Edits) (src : Name) (tgt : Import) : Edits :=
   match ed.get? src with
@@ -291,8 +407,8 @@ Returns `(path, inputCtx, imports, endPos)` where `imports` is the `Lean.Parser.
 and `endPos` is the position of the end of the header.
 -/
 def parseHeaderFromString (text path : String) :
-    IO (System.FilePath × Parser.InputContext ×
-      TSyntax ``Parser.Module.header × String.Pos.Raw) := do
+    IO (System.FilePath × (ictx : Parser.InputContext) ×
+      TSyntax ``Parser.Module.header × String.Pos ictx.fileMap.source) := do
   let inputCtx := Parser.mkInputContext text path
   let (header, parserState, msgs) ← Parser.parseHeader inputCtx
   if !msgs.toList.isEmpty then -- skip this file if there are parse errors
@@ -300,8 +416,8 @@ def parseHeaderFromString (text path : String) :
     throw <| .userError "parse errors in file"
   -- the insertion point for `add` is the first newline after the imports
   let insertion := header.raw.getTailPos?.getD parserState.pos
-  let insertion := text.findAux (· == '\n') text.rawEndPos insertion + '\n'
-  pure (path, inputCtx, header, insertion)
+  let insertion := inputCtx.fileMap.source.pos! insertion |>.find (· == '\n') |>.next!
+  pure ⟨path, inputCtx, header, insertion⟩
 
 /-- Parse a source file to extract the location of the import lines, for edits and error messages.
 
@@ -309,8 +425,8 @@ Returns `(path, inputCtx, imports, endPos)` where `imports` is the `Lean.Parser.
 and `endPos` is the position of the end of the header.
 -/
 def parseHeader (srcSearchPath : SearchPath) (mod : Name) :
-    IO (System.FilePath × Parser.InputContext ×
-      TSyntax ``Parser.Module.header × String.Pos.Raw) := do
+    IO (System.FilePath × (ictx : Parser.InputContext) ×
+      TSyntax ``Parser.Module.header × String.Pos ictx.fileMap.source) := do
   -- Parse the input file
   let some path ← srcSearchPath.findModuleWithExt "lean" mod
     | throw <| .userError s!"error: failed to find source file for {mod}"
@@ -320,7 +436,7 @@ def parseHeader (srcSearchPath : SearchPath) (mod : Name) :
 def decodeHeader : TSyntax ``Parser.Module.header → Option (TSyntax `module) × Option (TSyntax `prelude) × TSyntaxArray ``Parser.Module.import
   | `(Parser.Module.header| $[module%$moduleTk?]? $[prelude%$preludeTk?]? $imports*) =>
     (moduleTk?.map .mk, preludeTk?.map .mk, imports)
-  | _ => unreachable!
+  | stx => panic! s!"unexpected header syntax {stx}"
 
 def decodeImport : TSyntax ``Parser.Module.import → Import
   | `(Parser.Module.import| $[public%$pubTk?]? $[meta%$metaTk?]? import $[all%$allTk?]? $id) =>
@@ -329,73 +445,171 @@ def decodeImport : TSyntax ``Parser.Module.import → Import
 
 /-- Analyze and report issues from module `i`. Arguments:
 
+* `pkg`: the first component of the module name
 * `srcSearchPath`: Used to find the path for error reporting purposes
 * `i`: the module index
 * `needs`: the module's calculated needs
-* `pinned`: dependencies that should be preserved even if unused
-* `edits`: accumulates the list of edits to apply if `--fix` is true
 * `addOnly`: if true, only add missing imports, do not remove unused ones
 -/
-def visitModule (srcSearchPath : SearchPath)
-    (i : Nat) (needs : Needs) (preserve : Needs) (edits : Edits) (headerStx : TSyntax ``Parser.Module.header)
-    (addOnly := false) (githubStyle := false) (explain := false) : StateT State IO Edits := do
+def visitModule (pkg : Name) (srcSearchPath : SearchPath)
+    (i : Nat) (needs : Needs) (headerStx : TSyntax ``Parser.Module.header) (args : Args)
+    (addOnly := false) : StateT State IO Unit := do
+  if isExtraRevModUse (← get).env i then
+    modify fun s => { s with preserve := s.preserve.union (if args.addPublic then .pub else .priv) {i} }
+    if args.trace then
+      IO.eprintln s!"Preserving `{(← get).modNames[i]!}` because of recorded extra rev use"
+
+  -- only process modules in the selected package
+  -- TODO: should be after `keep-downstream` but core headers are not found yet?
+  if !pkg.isPrefixOf (← get).modNames[i]! then
+    return
+
+  let (module?, prelude?, imports) := decodeHeader headerStx
+  if module?.any (·.raw.getTrailing?.any (·.toString.contains "shake: keep-downstream")) then
+    modify fun s => { s with preserve := s.preserve.union .pub {i} }
+
   let s ← get
-  -- Do transitive reduction of `needs` in `deps`.
+
+  let addOnly := addOnly || module?.any (·.raw.getTrailing?.any (·.toString.contains "shake: keep-all"))
   let mut deps := needs
-  let (_, prelude?, imports) := decodeHeader headerStx
+
+  -- Add additional preserved imports
+  for impStx in imports do
+    let imp := decodeImport impStx
+    let j := s.env.getModuleIdx? imp.module |>.get!
+    let k := NeedsKind.ofImport imp
+    if addOnly ||
+        args.keepPublic && imp.isExported ||
+        impStx.raw.getTrailing?.any (·.toString.contains "shake: keep") then
+      deps := deps.union k {j}
+      if args.trace then
+        IO.eprintln s!"Adding `{imp}` as additional dependency"
+  for j in [0:s.mods.size] do
+    for k in NeedsKind.all do
+      -- remove `meta` while preserving, no use-case for preserving `meta` so far
+      if s.transDepsOrig[i]!.has k j && s.preserve.has { k with isMeta := false } j then
+        deps := deps.union { k with isMeta := false } {j}
+
+  -- Do transitive reduction of `needs` in `deps`.
+  if !addOnly then
+    for j in [0:s.mods.size] do
+      let transDeps := s.transDeps[j]!
+      for k in NeedsKind.all do
+        if deps.has k j then
+          let transDeps := addTransitiveImps .empty { k with module := .anonymous } j transDeps
+          for k' in NeedsKind.all do
+            deps := deps.sub k' (transDeps.sub k' {j} |>.get k')
+
   if prelude?.isNone then
     deps := deps.union .pub {s.env.getModuleIdx? `Init |>.get!}
-  for imp in imports do
-    if addOnly || imp.raw.getTrailing?.any (·.toString.toSlice.contains "shake: keep") then
-      let imp := decodeImport imp
-      let j := s.env.getModuleIdx? imp.module |>.get!
-      let k := NeedsKind.ofImport imp
-      deps := deps.union k {j}
-  for j in [0:s.mods.size] do
-    let transDeps := s.transDeps[j]!
-    for k in NeedsKind.all do
-      if s.transDepsOrig[i]!.has k j && preserve.has k j then
-        deps := deps.union k {j}
-      if deps.has k j then
-        let transDeps := addTransitiveImps .empty { k with module := .anonymous } j transDeps
-        for k' in NeedsKind.all do
-          deps := deps.sub k' (transDeps.sub k' {j} |>.get k')
 
-  -- Any import which is not in `transDeps` was unused.
-  -- Also accumulate `newDeps` which is the transitive closure of the remaining imports
-  let mut toRemove : Array Import := #[]
-  let mut newDeps := Needs.empty
+  -- Accumulate `transDeps` which is the non-reflexive transitive closure of the still-live imports
+  let mut transDeps := Needs.empty
+  let mut alwaysAdd : Array Import := #[]  -- to be added even if implied by other imports
   for imp in s.mods[i]!.imports do
     let j := s.env.getModuleIdx? imp.module |>.get!
-    if
-        -- skip folder-nested imports
-        s.modNames[i]!.isPrefixOf imp.module ||
-        imp.importAll then
-      newDeps := addTransitiveImps newDeps imp j s.transDeps[j]!
-    else
-      let k := NeedsKind.ofImport imp
-      -- A private import should also be removed if the public version is needed
-      if !deps.has k j || !k.isExported && deps.has { k with isExported := true } j then
-        toRemove := toRemove.push imp
-      else
-        newDeps := addTransitiveImps newDeps imp j s.transDeps[j]!
+    let k := NeedsKind.ofImport imp
+    if deps.has k j || imp.importAll then
+      transDeps := addTransitiveImps transDeps imp j s.transDeps[j]!
+      deps := deps.union k {j}
+    -- skip folder-nested `public (meta)? import`s but remove `meta`
+    else if s.modNames[i]!.isPrefixOf imp.module then
+      let imp := { imp with isMeta := false }
+      let k := { k with isMeta := false }
+      if args.trace then
+        IO.eprintln s!"`{imp}` is preserved as folder-nested import"
+      transDeps := addTransitiveImps transDeps imp j s.transDeps[j]!
+      deps := deps.union k {j}
+      if !s.mods[i]!.imports.contains imp then
+        alwaysAdd := alwaysAdd.push imp
 
-  -- If `newDeps` does not cover `deps`, then we have to add back some imports until it does.
+  -- If `transDeps` does not cover `deps`, then we have to add back some imports until it does.
   -- To minimize new imports we pick only new imports which are not transitively implied by
-  -- another new import
+  -- another new import, so we visit module indices in descending order.
+  let mut keptPrefix := false
+  let mut newTransDeps := transDeps
   let mut toAdd : Array Import := #[]
-  for j in [0:s.mods.size] do
+  for j in (0...s.mods.size).toArray.reverse do
     for k in NeedsKind.all do
-      if deps.has k j && !newDeps.has k j && !newDeps.has { k with isExported := true } j then
-        let imp := { k with module := s.modNames[j]! }
-        toAdd := toAdd.push imp
-        newDeps := addTransitiveImps newDeps imp j s.transDeps[j]!
+      if deps.has k j && !newTransDeps.has k j && !newTransDeps.has { k with isExported := true } j then
+        -- `add-public/keep-prefix` may change the import and even module we're considering
+        let mut k := k
+        let mut imp : Import := { k with module := s.modNames[j]! }
+        let mut j := j
+        if args.trace then
+          IO.eprintln s!"`{imp}` is needed"
+        if args.addPublic && !k.isExported &&
+            -- also add as public if previously `public meta`, which could be from automatic porting
+            (s.transDepsOrig[i]!.has { k with isExported := true } j || s.transDepsOrig[i]!.has { k with isExported := true, isMeta := true } j) then
+          k := { k with isExported := true }
+          imp := { imp with isExported := true }
+          if args.trace then
+            IO.eprintln s!"* upgrading to `{imp}` because of `--add-public`"
+        if args.keepPrefix then
+          let rec tryPrefix : Name → Option ModuleIdx
+            | .str p _ => tryPrefix p <|> (do
+              let j' ← s.env.getModuleIdx? p
+              -- `j'` must be reachable from `i` (allow downgrading from `meta`)
+              guard <| s.transDepsOrig[i]!.has k j' || s.transDepsOrig[i]!.has { k with isMeta := true } j'
+              let j'transDeps := addTransitiveImps .empty p j' s.transDeps[j']!
+              -- `j` must be reachable from `j'` (now downgrading must be done in the other direction)
+              guard <| j'transDeps.has k j || j'transDeps.has { k with isMeta := false } j
+              return j')
+            | _ => none
+          if let some j' := tryPrefix imp.module then
+            imp := { imp with module := s.modNames[j']! }
+            j := j'
+            keptPrefix := true
+            if args.trace then
+              IO.eprintln s!"* upgrading to `{imp}` because of `--keep-prefix`"
+        if !s.mods[i]!.imports.contains imp then
+          toAdd := toAdd.push imp
+        deps := deps.union k {j}
+        newTransDeps := addTransitiveImps newTransDeps imp j s.transDeps[j]!
+
+  if keptPrefix then
+    -- if an import was replaced by `--keep-prefix`, we did not necessarily visit the modules in
+    -- dependency order anymore and so we have to redo the transitive closure checking
+    newTransDeps := transDeps
+    for j in (0...s.mods.size).toArray.reverse do
+      for k in NeedsKind.all do
+        if deps.has k j then
+          let mut imp : Import := { k with module := s.modNames[j]! }
+          if toAdd.contains imp && (newTransDeps.has k j || newTransDeps.has { k with isExported := true } j) then
+            if args.trace then
+              IO.eprintln s!"Removing `{imp}` from imports to be added because it is now implied"
+            toAdd := toAdd.erase imp
+            deps := deps.sub k {j}
+          else
+            newTransDeps := addTransitiveImps newTransDeps imp j s.transDeps[j]!
+
+  -- now that `toAdd` filtering is done, add `alwaysAdd`
+  toAdd := alwaysAdd ++ toAdd
+
+  -- Any import which is still not in `deps` was unused
+  let mut toRemove : Array Import := #[]
+  for imp in s.mods[i]!.imports do
+    let j := s.env.getModuleIdx? imp.module |>.get!
+    let k := NeedsKind.ofImport imp
+    if args.keepImplied && newTransDeps.has k j then
+      if args.trace && !deps.has k j then
+        IO.eprintln s!"`{imp}` is implied by other imports"
+    else if !deps.has k j then
+      if args.trace then
+        IO.eprintln s!"`{imp}` is now unused"
+      toRemove := toRemove.push imp
+    -- A private import should also be removed if the public version has been added
+    else if !k.isExported && !imp.importAll && newTransDeps.has { k with isExported := true } j then
+      if args.trace then
+        IO.eprintln s!"`{imp}` is already covered by `{ { imp with isExported := true } }`"
+      toRemove := toRemove.push imp
 
   -- mark and report the removals
-  let mut edits := toRemove.foldl (init := edits) fun edits imp =>
-    edits.remove s.modNames[i]! imp
+  modify fun s => { s with
+    edits := toRemove.foldl (init := s.edits) fun edits imp =>
+      edits.remove s.modNames[i]! imp }
 
-  if !toAdd.isEmpty || !toRemove.isEmpty || explain then
+  if !toAdd.isEmpty || !toRemove.isEmpty || args.explain then
     if let some path ← srcSearchPath.findModuleWithExt "lean" s.modNames[i]! then
       println! "{path}:"
     else
@@ -404,9 +618,9 @@ def visitModule (srcSearchPath : SearchPath)
   if !toRemove.isEmpty then
     println! "  remove {toRemove}"
 
-  if githubStyle then
+  if args.githubStyle then
     try
-      let (path, inputCtx, stx, endHeader) ← parseHeader srcSearchPath s.modNames[i]!
+      let ⟨path, inputCtx, stx, endHeader⟩ ← parseHeader srcSearchPath s.modNames[i]!
       let (_, _, imports) := decodeHeader stx
       for stx in imports do
         if toRemove.any fun imp => imp == decodeImport stx then
@@ -415,14 +629,15 @@ def visitModule (srcSearchPath : SearchPath)
             (use `lake exe shake --fix` to fix this, or `lake exe shake --update` to ignore)"
       if !toAdd.isEmpty then
         -- we put the insert message on the beginning of the last import line
-        let pos := inputCtx.fileMap.toPosition endHeader
+        let pos := inputCtx.fileMap.toPosition endHeader.offset
         println! "{path}:{pos.line-1}:1: warning: \
           add {toAdd} instead"
     catch _ => pure ()
 
   -- mark and report the additions
-  edits := toAdd.foldl (init := edits) fun edits imp =>
-    edits.add s.modNames[i]! imp
+  modify fun s => { s with
+    edits := toAdd.foldl (init := s.edits) fun edits imp =>
+      edits.add s.modNames[i]! imp }
 
   if !toAdd.isEmpty then
     println! "  add {toAdd}"
@@ -437,14 +652,15 @@ def visitModule (srcSearchPath : SearchPath)
     let j := s.env.getModuleIdx? imp.module |>.get!
     newTransDepsI := addTransitiveImps newTransDepsI imp j s.transDeps[j]!
 
-  set { s with transDeps := s.transDeps.set! i newTransDepsI }
+  modify fun s => { s with transDeps := s.transDeps.set! i newTransDepsI }
 
-  if explain then
+  if args.explain then
     let explanation := getExplanations s.env i
     let sanitize n := if n.hasMacroScopes then (sanitizeName n).run' { options := {} } else n
     let run (imp : Import) := do
       let j := s.env.getModuleIdx? imp.module |>.get!
-      if let some exp? := explanation[(j, NeedsKind.ofImport imp)]? then
+      let mut k := NeedsKind.ofImport imp
+      if let some exp? := explanation[(j, k)]? <|> guard args.addPublic *> explanation[(j, { k with isExported := false})]? then
         println! "  note: `{imp}` required"
         if let some (n, c) := exp? then
           println! "    because `{sanitize n}` refers to `{sanitize c}`"
@@ -455,8 +671,6 @@ def visitModule (srcSearchPath : SearchPath)
         run j
     for i in toAdd do run i
 
-  return edits
-
 /-- Convert a list of module names to a bitset of module indexes -/
 def toBitset (s : State) (ns : List Name) : Bitset :=
   ns.foldl (init := ∅) fun c name =>
@@ -464,40 +678,26 @@ def toBitset (s : State) (ns : List Name) : Bitset :=
     | some i => c ∪ {i}
     | none => c
 
-/-- The parsed CLI arguments. See `help` for more information -/
-structure Args where
-  /-- `--help`: shows the help -/
-  help : Bool := false
-  /-- `--force`: skips the `lake build --no-build` sanity check -/
-  force : Bool := false
-  /-- `--gh-style`: output messages that can be parsed by `gh-problem-matcher-wrap` -/
-  githubStyle : Bool := false
-  /-- `--explain`: give constants explaining why each module is needed -/
-  explain : Bool := false
-  /-- `--fix`: apply the fixes directly -/
-  fix : Bool := false
-  /-- `<MODULE>..`: the list of root modules to check -/
-  mods : Array Name := #[]
-
 local instance : Ord Import where
-  compare a b :=
-    if a.isExported && !b.isExported then
-      Ordering.lt
-    else if !a.isExported && b.isExported then
-      Ordering.gt
-    else
-      a.module.cmp b.module
+  compare :=
+    let _ := @lexOrd
+    compareOn fun imp => (!imp.isExported, imp.module.toString)
 
 /-- The main entry point. See `help` for more information on arguments. -/
-def main (args : List String) : IO UInt32 := do
+public def main (args : List String) : IO UInt32 := do
   initSearchPath (← findSysroot)
   -- Parse the arguments
   let rec parseArgs (args : Args) : List String → Args
     | [] => args
     | "--help" :: rest => parseArgs { args with help := true } rest
+    | "--keep-implied" :: rest => parseArgs { args with keepImplied := true } rest
+    | "--keep-prefix" :: rest => parseArgs { args with keepPrefix := true } rest
+    | "--keep-public" :: rest => parseArgs { args with keepPublic := true } rest
+    | "--add-public" :: rest => parseArgs { args with addPublic := true } rest
     | "--force" :: rest => parseArgs { args with force := true } rest
     | "--fix" :: rest => parseArgs { args with fix := true } rest
     | "--explain" :: rest => parseArgs { args with explain := true } rest
+    | "--trace" :: rest => parseArgs { args with trace := true } rest
     | "--gh-style" :: rest => parseArgs { args with githubStyle := true } rest
     | "--" :: rest => { args with mods := args.mods ++ rest.map (·.toName) }
     | other :: rest => parseArgs { args with mods := args.mods.push other.toName } rest
@@ -540,69 +740,69 @@ def main (args : List String) : IO UInt32 := do
   let imps := mods.map ({ module := · })
   let (_, s) ← importModulesCore imps (isExported := true) |>.run
   let s := s.markAllExported
-  let env ← finalizeImport s (isModule := true) imps {} (leakEnv := false) (loadExts := false)
+  let mut env ← finalizeImport s (isModule := true) imps {} (leakEnv := false) (loadExts := false)
+  -- the one env ext we want to initialize
+  let is := indirectModUseExt.toEnvExtension.getState env
+  let newState ← indirectModUseExt.addImportedFn is.importedEntries { env := env, opts := {} }
+  env := indirectModUseExt.toEnvExtension.setState (asyncMode := .sync) env { is with state := newState }
 
   StateT.run' (s := initStateFromEnv env) do
 
   let s ← get
-  -- Parse the config file
-
   -- Run the calculation of the `needs` array in parallel
   let needs := s.mods.mapIdx fun i _ =>
-    Task.spawn fun _ => calcNeeds s.env i
+    Task.spawn fun _ => calcNeeds s i
 
   -- Parse headers in parallel
   let headers ← s.mods.mapIdxM fun i _ =>
-    BaseIO.asTask (parseHeader srcSearchPath s.modNames[i]! |>.toBaseIO)
+    if !pkg.isPrefixOf s.modNames[i]! then
+      pure <| Task.pure <| .ok ⟨default, default, default, default⟩
+    else
+      BaseIO.asTask (parseHeader srcSearchPath s.modNames[i]! |>.toBaseIO)
 
   if args.fix then
     println! "The following changes will be made automatically:"
 
   -- Check all selected modules
-  let mut edits : Edits := ∅
-  let mut revNeeds : Needs := default
   for i in [0:s.mods.size], t in needs, header in headers do
     match header.get with
-    | .ok (_, _, stx, _) =>
-      edits ← visitModule (addOnly := !pkg.isPrefixOf s.modNames[i]!)
-        srcSearchPath i t.get revNeeds edits stx args.githubStyle args.explain
-      if isExtraRevModUse s.env i then
-        revNeeds := revNeeds.union .priv {i}
+    | .ok ⟨_, _, stx, _⟩ =>
+      visitModule pkg srcSearchPath i t.get stx args
     | .error e =>
       println! e.toString
 
   if !args.fix then
     -- return error if any issues were found
-    return if edits.isEmpty then 0 else 1
+    return if (← get).edits.isEmpty then 0 else 1
 
   -- Apply the edits to existing files
   let mut count := 0
   for mod in s.modNames, header? in headers do
-    let some (remove, add) := edits[mod]? | continue
+    let some (remove, add) := (← get).edits[mod]? | continue
     let add : Array Import := add.qsortOrd
 
     -- Parse the input file
-    let .ok (path, inputCtx, stx, insertion) := header?.get | continue
+    let .ok ⟨path, inputCtx, stx, insertion⟩ := header?.get | continue
     let (_, _, imports) := decodeHeader stx
     let text := inputCtx.fileMap.source
 
     -- Calculate the edit result
-    let mut pos : String.Pos.Raw := 0
+    let mut pos : String.Pos text := text.startPos
     let mut out : String := ""
     let mut seen : Std.HashSet Import := {}
     for stx in imports do
       let mod := decodeImport stx
       if remove.contains mod || seen.contains mod then
-        out := out ++ String.Pos.Raw.extract text pos stx.raw.getPos?.get!
+        out := out ++ pos.extract (text.pos! stx.raw.getPos?.get!)
         -- We use the end position of the syntax, but include whitespace up to the first newline
-        pos := text.findAux (· == '\n') text.rawEndPos stx.raw.getTailPos?.get! + '\n'
+        pos := text.pos! stx.raw.getTailPos?.get! |>.find '\n' |>.next!
       seen := seen.insert mod
-    out := out ++ String.Pos.Raw.extract text pos insertion
+    out := out ++ pos.extract insertion
     for mod in add do
       if !seen.contains mod then
         seen := seen.insert mod
         out := out ++ s!"{mod}\n"
-    out := out ++ String.Pos.Raw.extract text insertion text.rawEndPos
+    out := out ++ insertion.extract text.endPos
 
     IO.FS.writeFile path out
     count := count + 1

script/lakefile.toml

@@ -1,4 +1,5 @@
 name = "scripts"
+leanOptions = { experimental.module = true }
 
 [[lean_exe]]
 name = "modulize"
@@ -7,3 +8,5 @@ root = "Modulize"
 [[lean_exe]]
 name = "shake"
 root = "Shake"
+# needed by `Lake.loadWorkspace`
+supportInterpreter = true

script/prepare-llvm-linux.sh

@@ -58,7 +58,11 @@ OPTIONS=()
 # We build cadical using the custom toolchain on Linux to avoid glibc versioning issues
 echo -n " -DLEAN_STANDALONE=ON -DCADICAL_USE_CUSTOM_CXX=ON"
 echo -n " -DCMAKE_CXX_COMPILER=$PWD/llvm-host/bin/clang++ -DLEAN_CXX_STDLIB='-Wl,-Bstatic -lc++ -lc++abi -Wl,-Bdynamic'"
-echo -n " -DLEAN_EXTRA_CXX_FLAGS='--sysroot $PWD/llvm -idirafter $GLIBC_DEV/include ${EXTRA_FLAGS:-}'"
+# these should also be used for cadical, so do not use `LEAN_EXTRA_CXX_FLAGS` here
+echo -n " -DCMAKE_CXX_FLAGS='--sysroot $PWD/llvm -idirafter $GLIBC_DEV/include ${EXTRA_FLAGS:-}'"
+# the above does not include linker flags which will be added below based on context, so skip the
+# generic check by cmake
+echo -n " -DCMAKE_C_COMPILER_WORKS=1 -DCMAKE_CXX_COMPILER_WORKS=1"
 # use target compiler directly when not cross-compiling
 if [[ -L llvm-host ]]; then
   echo -n " -DCMAKE_C_COMPILER=$PWD/stage1/bin/clang"

src/CMakeLists.txt

@@ -42,7 +42,7 @@ if(LLD_PATH)
 endif()
 
 set(LEAN_EXTRA_LINKER_FLAGS ${LEAN_EXTRA_LINKER_FLAGS_DEFAULT} CACHE STRING "Additional flags used by the linker")
-set(LEAN_EXTRA_CXX_FLAGS "" CACHE STRING "Additional flags used by the C++ compiler")
+set(LEAN_EXTRA_CXX_FLAGS "" CACHE STRING "Additional flags used by the C++ compiler. Unlike `CMAKE_CXX_FLAGS`, these will not be used to build e.g. cadical.")
 set(LEAN_TEST_VARS "LEAN_CC=${CMAKE_C_COMPILER}" CACHE STRING "Additional environment variables used when running tests")
 
 if (NOT CMAKE_BUILD_TYPE)
@@ -191,7 +191,7 @@ endif()
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/Modules")
 
 # Initialize CXXFLAGS.
-set(CMAKE_CXX_FLAGS                "${LEAN_EXTRA_CXX_FLAGS} -DLEAN_BUILD_TYPE=\"${CMAKE_BUILD_TYPE}\" -DLEAN_EXPORTING")
+set(CMAKE_CXX_FLAGS                "${CMAKE_CXX_FLAGS} ${LEAN_EXTRA_CXX_FLAGS} -DLEAN_BUILD_TYPE=\"${CMAKE_BUILD_TYPE}\" -DLEAN_EXPORTING")
 set(CMAKE_CXX_FLAGS_DEBUG          "-DLEAN_DEBUG")
 set(CMAKE_CXX_FLAGS_MINSIZEREL     "-DNDEBUG")
 set(CMAKE_CXX_FLAGS_RELEASE        "-DNDEBUG")
@@ -448,8 +448,8 @@ if(LLVM AND ${STAGE} GREATER 0)
     # - In particular, `host/bin/llvm-config` produces flags like `-Lllvm-host/lib/libLLVM`, while
     #   we need the path to be `-Lllvm/lib/libLLVM`. Thus, we perform this replacement here.
     string(REPLACE "llvm-host" "llvm" LEANSHARED_LINKER_FLAGS ${LEANSHARED_LINKER_FLAGS})
-    string(REPLACE "llvm-host" "llvm" LEAN_EXTRA_CXX_FLAGS ${LEAN_EXTRA_CXX_FLAGS})
-    message(VERBOSE "leanshared linker flags: '${LEANSHARED_LINKER_FLAGS}' | lean extra cxx flags '${LEAN_EXTR_CXX_FLAGS}'")
+    string(REPLACE "llvm-host" "llvm" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS})
+    message(VERBOSE "leanshared linker flags: '${LEANSHARED_LINKER_FLAGS}' | lean extra cxx flags '${CMAKE_CXX_FLAGS}'")
 endif()
 
 # get rid of unused parts of C++ stdlib

src/Init/Data/Array/Count.lean

@@ -62,12 +62,12 @@ theorem size_eq_countP_add_countP {xs : Array α} : xs.size = countP p xs + coun
   rcases xs with ⟨xs⟩
   simp [List.length_eq_countP_add_countP (p := p)]
 
-@[grind =]
 theorem countP_eq_size_filter {xs : Array α} : countP p xs = (filter p xs).size := by
   rcases xs with ⟨xs⟩
   simp [List.countP_eq_length_filter]
 
-@[grind =]
+grind_pattern countP_eq_size_filter => xs.countP p, xs.filter p
+
 theorem countP_eq_size_filter' : countP p = size ∘ filter p := by
   funext xs
   apply countP_eq_size_filter

src/Init/Data/Array/Lex/Lemmas.lean

@@ -73,19 +73,11 @@ private theorem cons_lex_cons [BEq α] {lt : α → α → Bool} {a b : α} {xs
        (lt a b || a == b && xs.lex ys lt) := by
   simp only [lex, size_append, List.size_toArray, List.length_cons, List.length_nil, Nat.zero_add,
     Nat.add_min_add_left, Nat.add_lt_add_iff_left, Std.Rco.forIn'_eq_forIn'_toList]
-  conv =>
-    lhs; congr; congr
-    rw [cons_lex_cons.forIn'_congr_aux Std.Rco.toList_eq_if_roo rfl (fun _ _ _ => rfl)]
-    simp only [bind_pure_comp, map_pure]
-    rw [cons_lex_cons.forIn'_congr_aux (if_pos (by omega)) rfl (fun _ _ _ => rfl)]
-  simp only [Std.toList_roo_eq_toList_rco_of_isSome_succ? (lo := 0) (h := rfl),
-    Std.PRange.UpwardEnumerable.succ?, Nat.add_comm 1, Std.PRange.Nat.toList_rco_succ_succ,
-    Option.get_some, List.forIn'_cons, List.size_toArray, List.length_cons, List.length_nil,
-    Nat.lt_add_one, getElem_append_left, List.getElem_toArray, List.getElem_cons_zero]
-  cases lt a b
-  · rw [bne]
-    cases a == b <;> simp
-  · simp
+  rw [cons_lex_cons.forIn'_congr_aux (Nat.toList_rco_eq_cons (by omega)) rfl (fun _ _ _ => rfl)]
+  simp only [bind_pure_comp, map_pure, Nat.toList_rco_succ_succ, Nat.add_comm 1]
+  cases h : lt a b
+  · cases h' : a == b <;> simp [bne, *]
+  · simp [*]
 
 @[simp, grind =] theorem _root_.List.lex_toArray [BEq α] {lt : α → α → Bool} {l₁ l₂ : List α} :
     l₁.toArray.lex l₂.toArray lt = l₁.lex l₂ lt := by

src/Init/Data/BitVec/Bitblast.lean

@@ -835,7 +835,7 @@ execution. -/
 structure DivModArgs (w : Nat) where
   /-- the numerator (aka, dividend) -/
   n : BitVec w
-  /-- the denumerator (aka, divisor)-/
+  /-- the denominator (aka, divisor)-/
   d : BitVec w
 
 /-- A `DivModState` is lawful if the remainder width `wr` plus the numerator width `wn` equals `w`,

src/Init/Data/BitVec/Lemmas.lean

@@ -5601,7 +5601,7 @@ theorem msb_eq_toNat {x : BitVec w}:
   simp only [msb_eq_decide, ge_iff_le]
 
 /-- Negating a bitvector created from a natural number equals
-creating a bitvector from the the negative of that number.
+creating a bitvector from the negative of that number.
 -/
 theorem neg_ofNat_eq_ofInt_neg {w : Nat} {x : Nat} :
     - BitVec.ofNat w x = BitVec.ofInt w (- x) := by

src/Init/Data/Bool.lean

@@ -260,7 +260,7 @@ instance : Std.Associative (· != ·) := ⟨bne_assoc⟩
 
 theorem eq_not_of_ne : ∀ {x y : Bool}, x ≠ y → x = !y := by decide
 
-/-! ### coercision related normal forms -/
+/-! ### coercion related normal forms -/
 
 theorem beq_eq_decide_eq [BEq α] [LawfulBEq α] [DecidableEq α] (a b : α) :
     (a == b) = decide (a = b) := by

src/Init/Data/FloatArray/Basic.lean

@@ -42,7 +42,7 @@ instance : EmptyCollection FloatArray where
 def push : FloatArray → Float → FloatArray
   | ⟨ds⟩, b => ⟨ds.push b⟩
 
-@[extern "lean_float_array_size"]
+@[extern "lean_float_array_size", tagged_return]
 def size : (@& FloatArray) → Nat
   | ⟨ds⟩ => ds.size
 

src/Init/Data/Int/Basic.lean

@@ -278,7 +278,11 @@ set_option bootstrap.genMatcherCode false in
 def decNonneg (m : @& Int) : Decidable (NonNeg m) :=
   match m with
   | ofNat m => isTrue <| NonNeg.mk m
-  | -[_ +1] => isFalse <| fun h => nomatch h
+  | -[i +1] => isFalse <| fun h =>
+    have : ∀ j, (j = -[i +1]) → NonNeg j → False := fun _ hj hnn =>
+      Int.NonNeg.casesOn (motive := fun j _ => j = -[i +1] → False) hnn
+        (fun _ h => Int.noConfusion h) hj
+    this -[i +1] rfl h
 
 /-- Decides whether `a ≤ b`.
 

src/Init/Data/Iterators/Basic.lean

@@ -393,6 +393,16 @@ abbrev IterM.Step {α : Type w} {m : Type w → Type w'} {β : Type w} [Iterator
     (it : IterM (α := α) m β) :=
   PlausibleIterStep it.IsPlausibleStep
 
+/--
+Makes a single step with the given iterator `it`, potentially emitting a value and providing a
+succeeding iterator. If this function is used recursively, termination can sometimes be proved with
+the termination measures `it.finitelyManySteps` and `it.finitelyManySkips`.
+-/
+@[always_inline, inline, expose]
+def IterM.step {α : Type w} {m : Type w → Type w'} {β : Type w} [Iterator α m β]
+    (it : IterM (α := α) m β) : m (Shrink it.Step) :=
+  Iterator.step it
+
 /--
 Asserts that a certain output value could plausibly be emitted by the given iterator in its next
 step.
@@ -420,16 +430,6 @@ def IterM.IsPlausibleSkipSuccessorOf {α : Type w} {m : Type w → Type w'} {β
     [Iterator α m β] (it' it : IterM (α := α) m β) : Prop :=
   it.IsPlausibleStep (.skip it')
 
-/--
-Makes a single step with the given iterator `it`, potentially emitting a value and providing a
-succeeding iterator. If this function is used recursively, termination can sometimes be proved with
-the termination measures `it.finitelyManySteps` and `it.finitelyManySkips`.
--/
-@[always_inline, inline, expose]
-def IterM.step {α : Type w} {m : Type w → Type w'} {β : Type w} [Iterator α m β]
-    (it : IterM (α := α) m β) : m (Shrink it.Step) :=
-  Iterator.step it
-
 end Monadic
 
 section Pure
@@ -717,6 +717,15 @@ def IterM.finitelyManySteps {α : Type w} {m : Type w → Type w'} {β : Type w}
     [Finite α m] (it : IterM (α := α) m β) : IterM.TerminationMeasures.Finite α m :=
   ⟨it⟩
 
+/--
+Termination measure to be used in well-founded recursive functions recursing over a finite iterator
+(see also `Finite`).
+-/
+@[expose]
+def IterM.finitelyManySteps! {α : Type w} {m : Type w → Type w'} {β : Type w} [Iterator α m β]
+    (it : IterM (α := α) m β) : IterM.TerminationMeasures.Finite α m :=
+  ⟨it⟩
+
 /--
 This theorem is used by a `decreasing_trivial` extension. It powers automatic termination proofs
 with `IterM.finitelyManySteps`.

src/Init/Data/Iterators/Combinators/Monadic/Attach.lean

@@ -91,21 +91,11 @@ instance Attach.instIteratorCollect {α β : Type w} {m : Type w → Type w'} [M
     IteratorCollect (Attach α m P) m n :=
   .defaultImplementation
 
-instance Attach.instIteratorCollectPartial {α β : Type w} {m : Type w → Type w'} [Monad m]
-    [Monad n] {P : β → Prop} [Iterator α m β] :
-    IteratorCollectPartial (Attach α m P) m n :=
-  .defaultImplementation
-
 instance Attach.instIteratorLoop {α β : Type w} {m : Type w → Type w'} [Monad m]
     {n : Type x → Type x'} [Monad n] {P : β → Prop} [Iterator α m β] :
     IteratorLoop (Attach α m P) m n :=
   .defaultImplementation
 
-instance Attach.instIteratorLoopPartial {α β : Type w} {m : Type w → Type w'} [Monad m]
-    {n : Type x → Type x'} [Monad n] {P : β → Prop} [Iterator α m β] :
-    IteratorLoopPartial (Attach α m P) m n :=
-  .defaultImplementation
-
 end Types
 
 /--

src/Init/Data/Iterators/Combinators/Monadic/FilterMap.lean

@@ -221,25 +221,11 @@ instance {α β γ : Type w} {m : Type w → Type w'}
     IteratorCollect (FilterMap α m n lift f) n o :=
   .defaultImplementation
 
-instance {α β γ : Type w} {m : Type w → Type w'}
-    {n : Type w → Type w''} {o : Type w → Type x} [Monad n] [Monad o] [Iterator α m β]
-    {lift : ⦃α : Type w⦄ → m α → n α}
-    {f : β → PostconditionT n (Option γ)} [Finite α m] :
-    IteratorCollectPartial (FilterMap α m n lift f) n o :=
-  .defaultImplementation
-
 instance FilterMap.instIteratorLoop {α β γ : Type w} {m : Type w → Type w'}
-    {n : Type w → Type w''} {o : Type x → Type x'}
-    [Monad n] [Monad o] [Iterator α m β] {lift : ⦃α : Type w⦄ → m α → n α}
-    {f : β → PostconditionT n (Option γ)} [Finite α m] :
-    IteratorLoop (FilterMap α m n lift f) n o :=
-  .defaultImplementation
-
-instance FilterMap.instIteratorLoopPartial {α β γ : Type w} {m : Type w → Type w'}
     {n : Type w → Type w''} {o : Type x → Type x'}
     [Monad n] [Monad o] [Iterator α m β] {lift : ⦃α : Type w⦄ → m α → n α}
     {f : β → PostconditionT n (Option γ)} :
-    IteratorLoopPartial (FilterMap α m n lift f) n o :=
+    IteratorLoop (FilterMap α m n lift f) n o :=
   .defaultImplementation
 
 /--
@@ -249,7 +235,7 @@ instance FilterMap.instIteratorLoopPartial {α β γ : Type w} {m : Type w → T
 instance Map.instIteratorCollect {α β γ : Type w} {m : Type w → Type w'}
     {n : Type w → Type w''} {o : Type w → Type x} [Monad n] [Monad o] [Iterator α m β]
     {lift₁ : ⦃α : Type w⦄ → m α → n α}
-    {f : β → PostconditionT n γ} [IteratorCollect α m o] [Finite α m] :
+    {f : β → PostconditionT n γ} [IteratorCollect α m o] :
     IteratorCollect (Map α m n lift₁ f) n o where
   toArrayMapped lift₂ _ g it :=
     letI : MonadLift m n := ⟨lift₁ (α := _)⟩
@@ -259,18 +245,6 @@ instance Map.instIteratorCollect {α β γ : Type w} {m : Type w → Type w'}
       (fun x => do g (← (f x).operation))
       it.internalState.inner (m := m)
 
-@[no_expose]
-instance Map.instIteratorCollectPartial {α β γ : Type w} {m : Type w → Type w'}
-    {n : Type w → Type w''} {o : Type w → Type x} [Monad n] [Monad o] [Iterator α m β]
-    {lift₁ : ⦃α : Type w⦄ → m α → n α}
-    {f : β → PostconditionT n γ} [IteratorCollectPartial α m o] :
-    IteratorCollectPartial (Map α m n lift₁ f) n o where
-  toArrayMappedPartial lift₂ _ g it :=
-    IteratorCollectPartial.toArrayMappedPartial
-      (lift := fun ⦃_⦄ a => lift₂ (lift₁ a))
-      (fun x => do g (← lift₂ (f x).operation))
-      it.internalState.inner (m := m)
-
 instance Map.instIteratorLoop {α β γ : Type w} {m : Type w → Type w'}
     {n : Type w → Type w''} {o : Type x → Type x'} [Monad n] [Monad o] [Iterator α m β]
     {lift : ⦃α : Type w⦄ → m α → n α}
@@ -278,13 +252,6 @@ instance Map.instIteratorLoop {α β γ : Type w} {m : Type w → Type w'}
     IteratorLoop (Map α m n lift f) n o :=
   .defaultImplementation
 
-instance Map.instIteratorLoopPartial {α β γ : Type w} {m : Type w → Type w'}
-    {n : Type w → Type w''} {o : Type x → Type x'} [Monad n] [Monad o] [Iterator α m β]
-    {lift : ⦃α : Type w⦄ → m α → n α}
-    {f : β → PostconditionT n γ} :
-    IteratorLoopPartial (Map α m n lift f) n o :=
-  .defaultImplementation
-
 /--
 *Note: This is a very general combinator that requires an advanced understanding of monads, dependent
 types and termination proofs. The variants `map` and `mapM` are easier to use and sufficient

src/Init/Data/Iterators/Combinators/Monadic/FlatMap.lean

@@ -33,7 +33,7 @@ public structure Flatten (α α₂ β : Type w) (m) where
 /--
 Internal iterator combinator that is used to implement all `flatMap` variants
 -/
-@[always_inline]
+@[always_inline, inline]
 def IterM.flattenAfter {α α₂ β : Type w} {m : Type w → Type w'} [Monad m]
     [Iterator α m (IterM (α := α₂) m β)] [Iterator α₂ m β]
     (it₁ : IterM (α := α) m (IterM (α := α₂) m β)) (it₂ : Option (IterM (α := α₂) m β)) :=
@@ -75,7 +75,7 @@ iterator.
 
 For each value emitted by the outer iterator `it₁`, this combinator calls `f`.
 -/
-@[always_inline]
+@[always_inline, inline]
 public def IterM.flatMapAfterM {α : Type w} {β : Type w} {α₂ : Type w}
     {γ : Type w} {m : Type w → Type w'} [Monad m] [Iterator α m β] [Iterator α₂ m γ]
     (f : β → m (IterM (α := α₂) m γ)) (it₁ : IterM (α := α) m β) (it₂ : Option (IterM (α := α₂) m γ)) :=
@@ -114,7 +114,7 @@ This combinator incurs an additional O(1) cost with each output of `it` or an in
 
 For each value emitted by the outer iterator `it`, this combinator calls `f`.
 -/
-@[always_inline, expose]
+@[always_inline, inline, expose]
 public def IterM.flatMapM {α : Type w} {β : Type w} {α₂ : Type w}
     {γ : Type w} {m : Type w → Type w'} [Monad m] [Iterator α m β] [Iterator α₂ m γ]
     (f : β → m (IterM (α := α₂) m γ)) (it : IterM (α := α) m β) :=
@@ -156,7 +156,7 @@ iterator.
 
 For each value emitted by the outer iterator `it₁`, this combinator calls `f`.
 -/
-@[always_inline]
+@[always_inline, inline]
 public def IterM.flatMapAfter {α : Type w} {β : Type w} {α₂ : Type w}
     {γ : Type w} {m : Type w → Type w'} [Monad m] [Iterator α m β] [Iterator α₂ m γ]
     (f : β → IterM (α := α₂) m γ) (it₁ : IterM (α := α) m β) (it₂ : Option (IterM (α := α₂) m γ)) :=
@@ -195,7 +195,7 @@ This combinator incurs an additional O(1) cost with each output of `it` or an in
 
 For each value emitted by the outer iterator `it`, this combinator calls `f`.
 -/
-@[always_inline, expose]
+@[always_inline, inline, expose]
 public def IterM.flatMap {α : Type w} {β : Type w} {α₂ : Type w}
     {γ : Type w} {m : Type w → Type w'} [Monad m] [Iterator α m β] [Iterator α₂ m γ]
     (f : β → IterM (α := α₂) m γ) (it : IterM (α := α) m β) :=
@@ -370,16 +370,8 @@ public instance Flatten.instIteratorCollect [Monad m] [Monad n] [Iterator α m (
     [Iterator α₂ m β] : IteratorCollect (Flatten α α₂ β m) m n :=
   .defaultImplementation
 
-public instance Flatten.instIteratorCollectPartial [Monad m] [Monad n] [Iterator α m (IterM (α := α₂) m β)]
-    [Iterator α₂ m β] : IteratorCollectPartial (Flatten α α₂ β m) m n :=
-  .defaultImplementation
-
 public instance Flatten.instIteratorLoop [Monad m] [Monad n] [Iterator α m (IterM (α := α₂) m β)]
     [Iterator α₂ m β] : IteratorLoop (Flatten α α₂ β m) m n :=
   .defaultImplementation
 
-public instance Flatten.instIteratorLoopPartial [Monad m] [Monad n] [Iterator α m (IterM (α := α₂) m β)]
-    [Iterator α₂ m β] : IteratorLoopPartial (Flatten α α₂ β m) m n :=
-  .defaultImplementation
-
 end Std.Iterators

src/Init/Data/Iterators/Combinators/Monadic/Take.lean

@@ -208,16 +208,8 @@ instance Take.instIteratorCollect {n : Type w → Type w'} [Monad m] [Monad n] [
     IteratorCollect (Take α m) m n :=
   .defaultImplementation
 
-instance Take.instIteratorCollectPartial {n : Type w → Type w'} [Monad m] [Monad n] [Iterator α m β] :
-    IteratorCollectPartial (Take α m) m n :=
-  .defaultImplementation
-
 instance Take.instIteratorLoop {n : Type x → Type x'} [Monad m] [Monad n] [Iterator α m β] :
     IteratorLoop (Take α m) m n :=
   .defaultImplementation
 
-instance Take.instIteratorLoopPartial [Monad m] [Monad n] [Iterator α m β] :
-    IteratorLoopPartial (Take α m) m n :=
-  .defaultImplementation
-
 end Std.Iterators

src/Init/Data/Iterators/Combinators/Monadic/ULift.lean

@@ -128,18 +128,10 @@ instance Types.ULiftIterator.instIteratorLoop {o : Type x → Type x'} [Monad n]
     IteratorLoop (ULiftIterator α m n β lift) n o :=
   .defaultImplementation
 
-instance Types.ULiftIterator.instIteratorLoopPartial {o : Type x → Type x'} [Monad n] [Monad o] [Iterator α m β] :
-    IteratorLoopPartial (ULiftIterator α m n β lift) n o :=
-  .defaultImplementation
-
 instance Types.ULiftIterator.instIteratorCollect [Monad n] [Monad o] [Iterator α m β] :
     IteratorCollect (ULiftIterator α m n β lift) n o :=
   .defaultImplementation
 
-instance Types.ULiftIterator.instIteratorCollectPartial {o} [Monad n] [Monad o] [Iterator α m β] :
-    IteratorCollectPartial (ULiftIterator α m n β lift) n o :=
-  .defaultImplementation
-
 /--
 Transforms an `m`-monadic iterator with values in `β` into an `n`-monadic iterator with
 values in `ULift β`. Requires a `MonadLift m (ULiftT n)` instance.

src/Init/Data/Iterators/Consumers.lean

@@ -11,5 +11,6 @@ public import Init.Data.Iterators.Consumers.Access
 public import Init.Data.Iterators.Consumers.Collect
 public import Init.Data.Iterators.Consumers.Loop
 public import Init.Data.Iterators.Consumers.Partial
+public import Init.Data.Iterators.Consumers.Total
 
 public import Init.Data.Iterators.Consumers.Stream

src/Init/Data/Iterators/Consumers/Collect.lean

@@ -7,6 +7,7 @@ module
 
 prelude
 public import Init.Data.Iterators.Consumers.Partial
+public import Init.Data.Iterators.Consumers.Total
 public import Init.Data.Iterators.Consumers.Monadic.Collect
 
 @[expose] public section
@@ -21,40 +22,113 @@ Concretely, the following operations are provided:
 * `Iter.toListRev`, collecting the values in a list in reverse order but more efficiently
 * `Iter.toArray`, collecting the values in an array
 
-Some operations are implemented using the `IteratorCollect` and `IteratorCollectPartial`
-typeclasses.
+Some operations are implemented using the `IteratorCollect` type class.
 -/
 
 namespace Std.Iterators
 
-@[always_inline, inline, inherit_doc IterM.toArray]
+/--
+Traverses the given iterator and stores the emitted values in an array.
+
+If the iterator is not finite, this function might run forever. The variant
+`it.ensureTermination.toArray` always terminates after finitely many steps.
+-/
+@[always_inline, inline]
 def Iter.toArray {α : Type w} {β : Type w}
-    [Iterator α Id β] [Finite α Id] [IteratorCollect α Id Id] (it : Iter (α := α) β) : Array β :=
+    [Iterator α Id β] [IteratorCollect α Id Id] (it : Iter (α := α) β) : Array β :=
   it.toIterM.toArray.run
 
-@[always_inline, inline, inherit_doc IterM.Partial.toArray]
-def Iter.Partial.toArray {α : Type w} {β : Type w}
-    [Iterator α Id β] [IteratorCollectPartial α Id Id] (it : Iter.Partial (α := α) β) : Array β :=
-  it.it.toIterM.allowNontermination.toArray.run
+/--
+Traverses the given iterator and stores the emitted values in an array.
 
-@[always_inline, inline, inherit_doc IterM.toListRev]
+This function is deprecated. Instead of `it.allowNontermination.toArray`, use `it.toArray`.
+-/
+@[always_inline, inline, deprecated Iter.toArray (since := "2025-12-04")]
+def Iter.Partial.toArray {α : Type w} {β : Type w}
+    [Iterator α Id β] [IteratorCollect α Id Id] (it : Iter.Partial (α := α) β) : Array β :=
+  it.it.toArray
+
+/--
+Traverses the given iterator and stores the emitted values in an array.
+
+This variant terminates after finitely many steps and requires a proof that the iterator is
+finite. If such a proof is not available, consider using `Iter.toArray`.
+-/
+@[always_inline, inline]
+def Iter.Total.toArray {α : Type w} {β : Type w}
+    [Iterator α Id β] [Finite α Id] [IteratorCollect α Id Id] (it : Iter.Total (α := α) β) :
+    Array β :=
+  it.it.toArray
+
+/--
+Traverses the given iterator and stores the emitted values in reverse order in a list. Because
+lists are prepend-only, this `toListRev` is usually more efficient that `toList`.
+
+If the iterator is not finite, this function might run forever. The variant
+`it.ensureTermination.toListRev` always terminates after finitely many steps.
+-/
+@[always_inline, inline]
 def Iter.toListRev {α : Type w} {β : Type w}
-    [Iterator α Id β] [Finite α Id] (it : Iter (α := α) β) : List β :=
+    [Iterator α Id β] (it : Iter (α := α) β) : List β :=
   it.toIterM.toListRev.run
 
-@[always_inline, inline, inherit_doc IterM.Partial.toListRev]
+/--
+Traverses the given iterator and stores the emitted values in reverse order in a list. Because
+lists are prepend-only, this `toListRev` is usually more efficient that `toList`.
+
+This function is deprecated. Instead of `it.allowNontermination.toListRev`, use `it.toListRev`.
+-/
+@[always_inline, inline, deprecated Iter.toListRev (since := "2025-12-04")]
 def Iter.Partial.toListRev {α : Type w} {β : Type w}
     [Iterator α Id β] (it : Iter.Partial (α := α) β) : List β :=
-  it.it.toIterM.allowNontermination.toListRev.run
+  it.it.toListRev
 
-@[always_inline, inline, inherit_doc IterM.toList]
+/--
+Traverses the given iterator and stores the emitted values in reverse order in a list. Because
+lists are prepend-only, this `toListRev` is usually more efficient that `toList`.
+
+This variant terminates after finitely many steps and requires a proof that the iterator is
+finite. If such a proof is not available, consider using `Iter.toListRev`.
+-/
+@[always_inline, inline]
+def Iter.Total.toListRev {α : Type w} {β : Type w}
+    [Iterator α Id β] [Finite α Id] (it : Iter.Total (α := α) β) : List β :=
+  it.it.toListRev
+
+/--
+Traverses the given iterator and stores the emitted values in a list. Because
+lists are prepend-only, `toListRev` is usually more efficient that `toList`.
+
+If the iterator is not finite, this function might run forever. The variant
+`it.ensureTermination.toList` always terminates after finitely many steps.
+-/
+@[always_inline, inline]
 def Iter.toList {α : Type w} {β : Type w}
-    [Iterator α Id β] [Finite α Id] [IteratorCollect α Id Id] (it : Iter (α := α) β) : List β :=
+    [Iterator α Id β] [IteratorCollect α Id Id] (it : Iter (α := α) β) : List β :=
   it.toIterM.toList.run
 
-@[always_inline, inline, inherit_doc IterM.Partial.toList]
+/--
+Traverses the given iterator and stores the emitted values in a list. Because
+lists are prepend-only, `toListRev` is usually more efficient that `toList`.
+
+This function is deprecated. Instead of `it.allowNontermination.toList`, use `it.toList`.
+-/
+@[always_inline, deprecated Iter.toList (since := "2025-12-04")]
 def Iter.Partial.toList {α : Type w} {β : Type w}
-    [Iterator α Id β] [IteratorCollectPartial α Id Id] (it : Iter.Partial (α := α) β) : List β :=
-  it.it.toIterM.allowNontermination.toList.run
+    [Iterator α Id β] [IteratorCollect α Id Id] (it : Iter.Partial (α := α) β) : List β :=
+  it.it.toList
+
+/--
+Traverses the given iterator and stores the emitted values in a list. Because
+lists are prepend-only, `toListRev` is usually more efficient that `toList`.
+
+This variant terminates after finitely many steps and requires a proof that the iterator is
+finite. If such a proof is not available, consider using `Iter.toList`.
+-/
+@[always_inline, inline]
+def Iter.Total.toList {α : Type w} {β : Type w}
+    [Iterator α Id β] [Finite α Id] [IteratorCollect α Id Id] (it : Iter.Total (α := α) β) :
+    List β :=
+  it.it.toList
 
 end Std.Iterators

src/Init/Data/Iterators/Consumers/Loop.lean

@@ -23,7 +23,7 @@ function in every iteration. Concretely, the following operations are provided:
 * `Iter.fold`, the analogue of `List.foldl`
 * `Iter.foldM`, the analogue of `List.foldlM`
 
-These operations are implemented using the `IteratorLoop` and `IteratorLoopPartial` typeclasses.
+These operations are implemented using the `IteratorLoop` type class.
 -/
 
 namespace Std.Iterators
@@ -35,7 +35,7 @@ or future library improvements will make it more comfortable.
 -/
 @[always_inline, inline]
 def Iter.instForIn' {α : Type w} {β : Type w} {n : Type x → Type x'} [Monad n]
-    [Iterator α Id β] [Finite α Id] [IteratorLoop α Id n] :
+    [Iterator α Id β] [IteratorLoop α Id n] :
     ForIn' n (Iter (α := α) β) β ⟨fun it out => it.IsPlausibleIndirectOutput out⟩ where
   forIn' it init f :=
     IteratorLoop.finiteForIn' (fun _ _ f c => f c.run) |>.forIn' it.toIterM init
@@ -43,7 +43,7 @@ def Iter.instForIn' {α : Type w} {β : Type w} {n : Type x → Type x'} [Monad
           f out (Iter.isPlausibleIndirectOutput_iff_isPlausibleIndirectOutput_toIterM.mpr h) acc
 
 instance (α : Type w) (β : Type w) (n : Type x → Type x') [Monad n]
-    [Iterator α Id β] [Finite α Id] [IteratorLoop α Id n] :
+    [Iterator α Id β] [IteratorLoop α Id n] :
     ForIn n (Iter (α := α) β) β :=
   haveI : ForIn' n (Iter (α := α) β) β _ := Iter.instForIn'
   instForInOfForIn'
@@ -53,44 +53,58 @@ An implementation of `for h : ... in ... do ...` notation for partial iterators.
 -/
 @[always_inline, inline]
 def Iter.Partial.instForIn' {α : Type w} {β : Type w} {n : Type x → Type x'} [Monad n]
-    [Iterator α Id β] [IteratorLoopPartial α Id n] :
+    [Iterator α Id β] [IteratorLoop α Id n] :
     ForIn' n (Iter.Partial (α := α) β) β ⟨fun it out => it.it.IsPlausibleIndirectOutput out⟩ where
   forIn' it init f :=
-    IteratorLoopPartial.forInPartial (α := α) (m := Id) (n := n) (fun _ _ f c => f c.run)
-      it.it.toIterM init
-      fun out h acc =>
-        f out (Iter.isPlausibleIndirectOutput_iff_isPlausibleIndirectOutput_toIterM.mpr h) acc
+    haveI := @Iter.instForIn'
+    forIn' it.it init f
 
 instance (α : Type w) (β : Type w) (n : Type x → Type x') [Monad n]
-    [Iterator α Id β] [IteratorLoopPartial α Id n] :
+    [Iterator α Id β] [IteratorLoop α Id n] :
     ForIn n (Iter.Partial (α := α) β) β :=
   haveI : ForIn' n (Iter.Partial (α := α) β) β _ := Iter.Partial.instForIn'
   instForInOfForIn'
 
+/--
+A `ForIn'` instance for iterators that is guaranteed to terminate after finitely many steps.
+It is not marked as an instance because the membership predicate is difficult to work with.
+-/
+@[always_inline, inline]
+def Iter.Total.instForIn' {α : Type w} {β : Type w} {n : Type x → Type x'} [Monad n]
+    [Iterator α Id β] [IteratorLoop α Id n] [Finite α Id] :
+    ForIn' n (Iter.Total (α := α) β) β ⟨fun it out => it.it.IsPlausibleIndirectOutput out⟩ where
+  forIn' it init f := Iter.instForIn'.forIn' it.it init f
+
+instance (α : Type w) (β : Type w) (n : Type x → Type x') [Monad n]
+    [Iterator α Id β] [IteratorLoop α Id n] [Finite α Id] :
+    ForIn n (Iter.Total (α := α) β) β :=
+  haveI : ForIn' n (Iter.Total (α := α) β) β _ := Iter.Total.instForIn'
+  instForInOfForIn'
+
 instance {m : Type x → Type x'}
-    {α : Type w} {β : Type w} [Iterator α Id β] [Finite α Id] [IteratorLoop α Id m] [Monad m] :
+    {α : Type w} {β : Type w} [Iterator α Id β] [IteratorLoop α Id m] [Monad m] :
     ForM m (Iter (α := α) β) β where
   forM it f := forIn it PUnit.unit (fun out _ => do f out; return .yield .unit)
 
 instance {m : Type x → Type x'}
-    {α : Type w} {β : Type w} [Iterator α Id β] [Finite α Id] [IteratorLoopPartial α Id m] [Monad m] :
+    {α : Type w} {β : Type w} [Iterator α Id β] [IteratorLoop α Id m] [Monad m] :
     ForM m (Iter.Partial (α := α) β) β where
   forM it f := forIn it PUnit.unit (fun out _ => do f out; return .yield .unit)
 
+instance {m : Type x → Type x'}
+    {α : Type w} {β : Type w} [Monad m] [Iterator α Id β] [IteratorLoop α Id m] [Finite α Id] :
+    ForM m (Iter.Total (α := α) β) β where
+  forM it f := forIn it PUnit.unit (fun out _ => do f out; return .yield .unit)
+
 /--
 Folds a monadic function over an iterator from the left, accumulating a value starting with `init`.
 The accumulated value is combined with the each element of the list in order, using `f`.
 
 It is equivalent to `it.toList.foldlM`.
-
-This function requires a `Finite` instance proving that the iterator will finish after a finite
-number of steps. If the iterator is not finite or such an instance is not available, consider using
-`it.allowNontermination.foldM` instead of `it.foldM`. However, it is not possible to formally
-verify the behavior of the partial variant.
 -/
 @[always_inline, inline]
 def Iter.foldM {m : Type x → Type x'} [Monad m]
-    {α : Type w} {β : Type w} {γ : Type x} [Iterator α Id β] [Finite α Id]
+    {α : Type w} {β : Type w} {γ : Type x} [Iterator α Id β]
     [IteratorLoop α Id m] (f : γ → β → m γ)
     (init : γ) (it : Iter (α := α) β) : m γ :=
   ForIn.forIn it init (fun x acc => ForInStep.yield <$> f acc x)
@@ -101,29 +115,39 @@ The accumulated value is combined with the each element of the list in order, us
 
 It is equivalent to `it.toList.foldlM`.
 
-This is a partial, potentially nonterminating, function. It is not possible to formally verify
-its behavior. If the iterator has a `Finite` instance, consider using `IterM.foldM` instead.
+This function is deprecated. Instead of `it.allowNontermination.foldM`, use `it.foldM`.
 -/
-@[always_inline, inline]
+@[always_inline, inline, deprecated Iter.foldM (since := "2025-12-04")]
 def Iter.Partial.foldM {m : Type x → Type x'} [Monad m]
     {α : Type w} {β : Type w} {γ : Type x} [Iterator α Id β]
-    [IteratorLoopPartial α Id m] (f : γ → β → m γ)
+    [IteratorLoop α Id m] (f : γ → β → m γ)
     (init : γ) (it : Iter.Partial (α := α) β) : m γ :=
-  ForIn.forIn it init (fun x acc => ForInStep.yield <$> f acc x)
+  it.it.foldM (init := init) f
+
+/--
+Folds a monadic function over an iterator from the left, accumulating a value starting with `init`.
+The accumulated value is combined with the each element of the list in order, using `f`.
+
+It is equivalent to `it.toList.foldlM`.
+
+This variant terminates after finitely many steps and requires a proof that the iterator is
+finite. If such a proof is not available, consider using `Iter.foldM`.
+-/
+@[always_inline, inline]
+def Iter.Total.foldM {m : Type x → Type x'} [Monad m]
+    {α : Type w} {β : Type w} {γ : Type x} [Iterator α Id β]
+    [IteratorLoop α Id m] [Finite α Id] (f : γ → β → m γ)
+    (init : γ) (it : Iter.Total (α := α) β) : m γ :=
+  it.it.foldM (init := init) f
 
 /--
 Folds a function over an iterator from the left, accumulating a value starting with `init`.
 The accumulated value is combined with the each element of the list in order, using `f`.
 
 It is equivalent to `it.toList.foldl`.
-
-This function requires a `Finite` instance proving that the iterator will finish after a finite
-number of steps. If the iterator is not finite or such an instance is not available, consider using
-`it.allowNontermination.fold` instead of `it.fold`. However, it is not possible to formally
-verify the behavior of the partial variant.
 -/
 @[always_inline, inline]
-def Iter.fold {α : Type w} {β : Type w} {γ : Type x} [Iterator α Id β] [Finite α Id]
+def Iter.fold {α : Type w} {β : Type w} {γ : Type x} [Iterator α Id β]
     [IteratorLoop α Id Id] (f : γ → β → γ)
     (init : γ) (it : Iter (α := α) β) : γ :=
   ForIn.forIn (m := Id) it init (fun x acc => ForInStep.yield (f acc x))
@@ -134,14 +158,28 @@ The accumulated value is combined with the each element of the list in order, us
 
 It is equivalent to `it.toList.foldl`.
 
-This is a partial, potentially nonterminating, function. It is not possible to formally verify
-its behavior. If the iterator has a `Finite` instance, consider using `IterM.fold` instead.
+This function is deprecated. Instead of `it.allowNontermination.fold`, use `it.fold`.
+-/
+@[always_inline, inline, deprecated Iter.fold (since := "2025-12-04")]
+def Iter.Partial.fold {α : Type w} {β : Type w} {γ : Type x} [Iterator α Id β]
+    [IteratorLoop α Id Id] (f : γ → β → γ)
+    (init : γ) (it : Iter.Partial (α := α) β) : γ :=
+  it.it.fold (init := init) f
+
+/--
+Folds a function over an iterator from the left, accumulating a value starting with `init`.
+The accumulated value is combined with the each element of the list in order, using `f`.
+
+It is equivalent to `it.toList.foldl`.
+
+This variant terminates after finitely many steps and requires a proof that the iterator is
+finite. If such a proof is not available, consider using `Iter.fold`.
 -/
 @[always_inline, inline]
-def Iter.Partial.fold {α : Type w} {β : Type w} {γ : Type x} [Iterator α Id β]
-    [IteratorLoopPartial α Id Id] (f : γ → β → γ)
-    (init : γ) (it : Iter.Partial (α := α) β) : γ :=
-  ForIn.forIn (m := Id) it init (fun x acc => ForInStep.yield (f acc x))
+def Iter.Total.fold {α : Type w} {β : Type w} {γ : Type x} [Iterator α Id β]
+    [IteratorLoop α Id Id] [Finite α Id] (f : γ → β → γ)
+    (init : γ) (it : Iter.Total (α := α) β) : γ :=
+  it.it.fold (init := init) f
 
 set_option doc.verso true in
 /--
@@ -151,9 +189,9 @@ any element emitted by the iterator {name}`it`.
 {lit}`O(|xs|)`. Short-circuits upon encountering the first match. The elements in {name}`it` are
 examined in order of iteration.
 -/
-@[specialize]
+@[always_inline]
 def Iter.anyM {α β : Type w} {m : Type → Type w'} [Monad m]
-    [Iterator α Id β] [IteratorLoop α Id m] [Finite α Id]
+    [Iterator α Id β] [IteratorLoop α Id m]
     (p : β → m Bool) (it : Iter (α := α) β) : m Bool :=
   ForIn.forIn it false (fun x _ => do
     if ← p x then
@@ -161,6 +199,23 @@ def Iter.anyM {α β : Type w} {m : Type → Type w'} [Monad m]
     else
       return .yield false)
 
+set_option doc.verso true in
+/--
+Returns {lean}`true` if the monadic predicate {name}`p` returns {lean}`true` for
+any element emitted by the iterator {name}`it`.
+
+{lit}`O(|xs|)`. Short-circuits upon encountering the first match. The elements in {name}`it` are
+examined in order of iteration.
+
+This variant terminates after finitely many steps and requires a proof that the iterator is
+finite. If such a proof is not available, consider using {name}`Iter.anyM`.
+-/
+@[always_inline, inline]
+def Iter.Total.anyM {α β : Type w} {m : Type → Type w'} [Monad m]
+    [Iterator α Id β] [IteratorLoop α Id m] [Finite α Id]
+    (p : β → m Bool) (it : Iter.Total (α := α) β) : m Bool :=
+  it.it.anyM p
+
 set_option doc.verso true in
 /--
 Returns {lean}`true` if the pure predicate {name}`p` returns {lean}`true` for
@@ -171,21 +226,38 @@ examined in order of iteration.
 -/
 @[inline]
 def Iter.any {α β : Type w}
-    [Iterator α Id β] [IteratorLoop α Id Id] [Finite α Id]
+    [Iterator α Id β] [IteratorLoop α Id Id]
     (p : β → Bool) (it : Iter (α := α) β) : Bool :=
   (it.anyM (fun x => pure (f := Id) (p x))).run
 
 set_option doc.verso true in
 /--
-Returns {lean}`true` if the monadic predicate {name}`p` returns {lean}`true` for
-all elements emitted by the iterator {name}`it`.
+Returns {lean}`true` if the pure predicate {name}`p` returns {lean}`true` for
+any element emitted by the iterator {name}`it`.
 
-{lit}`O(|xs|)`. Short-circuits upon encountering the first mismatch. The elements in {name}`it` are
+{lit}`O(|xs|)`. Short-circuits upon encountering the first match. The elements in {name}`it` are
+examined in order of iteration.
+
+This variant terminates after finitely many steps and requires a proof that the iterator is
+finite. If such a proof is not available, consider using {name}`Iter.any`.
+-/
+@[inline]
+def Iter.Total.any {α β : Type w}
+    [Iterator α Id β] [IteratorLoop α Id Id] [Finite α Id]
+    (p : β → Bool) (it : Iter.Total (α := α) β) : Bool :=
+  it.it.any p
+
+set_option doc.verso true in
+/--
+Returns {lean}`true` if the monadic predicate {name}`p` returns {lean}`true` for
+all element emitted by the iterator {name}`it`.
+
+{lit}`O(|xs|)`. Short-circuits upon encountering the first match. The elements in {name}`it` are
 examined in order of iteration.
 -/
-@[specialize]
+@[always_inline, inline]
 def Iter.allM {α β : Type w} {m : Type → Type w'} [Monad m]
-    [Iterator α Id β] [IteratorLoop α Id m] [Finite α Id]
+    [Iterator α Id β] [IteratorLoop α Id m]
     (p : β → m Bool) (it : Iter (α := α) β) : m Bool :=
   ForIn.forIn it true (fun x _ => do
     if ← p x then
@@ -195,36 +267,82 @@ def Iter.allM {α β : Type w} {m : Type → Type w'} [Monad m]
 
 set_option doc.verso true in
 /--
-Returns {lean}`true` if the pure predicate {name}`p` returns {lean}`true` for
-all elements emitted by the iterator {name}`it`.
+Returns {lean}`true` if the monadic predicate {name}`p` returns {lean}`true` for
+all element emitted by the iterator {name}`it`.
 
-{lit}`O(|xs|)`. Short-circuits upon encountering the first mismatch. The elements in {name}`it` are
+{lit}`O(|xs|)`. Short-circuits upon encountering the first match. The elements in {name}`it` are
+examined in order of iteration.
+
+This variant terminates after finitely mall steps and requires a proof that the iterator is
+finite. If such a proof is not available, consider using {name}`Iter.allM`.
+-/
+@[always_inline, inline]
+def Iter.Total.allM {α β : Type w} {m : Type → Type w'} [Monad m]
+    [Iterator α Id β] [IteratorLoop α Id m] [Finite α Id]
+    (p : β → m Bool) (it : Iter.Total (α := α) β) : m Bool :=
+  it.it.allM p
+
+set_option doc.verso true in
+/--
+Returns {lean}`true` if the pure predicate {name}`p` returns {lean}`true` for
+all element emitted by the iterator {name}`it`.
+
+{lit}`O(|xs|)`. Short-circuits upon encountering the first match. The elements in {name}`it` are
 examined in order of iteration.
 -/
 @[inline]
 def Iter.all {α β : Type w}
-    [Iterator α Id β] [IteratorLoop α Id Id] [Finite α Id]
+    [Iterator α Id β] [IteratorLoop α Id Id]
     (p : β → Bool) (it : Iter (α := α) β) : Bool :=
   (it.allM (fun x => pure (f := Id) (p x))).run
 
+set_option doc.verso true in
 /--
-Steps through the iterator until the monadic function `f` returns `some` for an element, at which
-point iteration stops and the result of `f` is returned. If the iterator is completely consumed
-without `f` returning `some`, then the result is `none`.
+Returns {lean}`true` if the pure predicate {name}`p` returns {lean}`true` for
+all element emitted by the iterator {name}`it`.
+
+{lit}`O(|xs|)`. Short-circuits upon encountering the first match. The elements in {name}`it` are
+examined in order of iteration.
+
+This variant terminates after finitely mall steps and requires a proof that the iterator is
+finite. If such a proof is not available, consider using {name}`Iter.all`.
+-/
+@[inline]
+def Iter.Total.all {α β : Type w}
+    [Iterator α Id β] [IteratorLoop α Id Id] [Finite α Id]
+    (p : β → Bool) (it : Iter.Total (α := α) β) : Bool :=
+  it.it.all p
+
+/--
+Returns the first non-`none` result of applying the monadic function `f` to each output
+of the iterator, in order. Returns `none` if `f` returns `none` for all outputs.
+
+`O(|it|)`. Short-circuits when `f` returns `some _`. The outputs of `it` are
+examined in order of iteration.
+
+If the iterator is not finite, this function might run forever. The variant
+`it.ensureTermination.findSomeM?` always terminates after finitely many steps.
+
+Example:
+```lean example
+#eval [7, 6, 5, 8, 1, 2, 6].iter.findSomeM? fun i => do
+  if i < 5 then
+    return some (i * 10)
+  if i ≤ 6 then
+    IO.println s!"Almost! {i}"
+  return none
+```
+```output
+Almost! 6
+Almost! 5
+```
+```output
+some 10
+```
 -/
 @[inline]
 def Iter.findSomeM? {α β : Type w} {γ : Type x} {m : Type x → Type w'} [Monad m] [Iterator α Id β]
-    [IteratorLoop α Id m] [Finite α Id] (it : Iter (α := α) β) (f : β → m (Option γ)) :
-    m (Option γ) :=
-  ForIn.forIn it none (fun x _ => do
-    match ← f x with
-    | none => return .yield none
-    | some fx => return .done (some fx))
-
-@[inline, inherit_doc Iter.findSomeM?]
-def Iter.Partial.findSomeM? {α β : Type w} {γ : Type x} {m : Type x → Type w'} [Monad m]
-    [Iterator α Id β] [IteratorLoopPartial α Id m] (it : Iter.Partial (α := α) β)
-    (f : β → m (Option γ)) :
+    [IteratorLoop α Id m] (it : Iter (α := α) β) (f : β → m (Option γ)) :
     m (Option γ) :=
   ForIn.forIn it none (fun x _ => do
     match ← f x with
@@ -232,52 +350,284 @@ def Iter.Partial.findSomeM? {α β : Type w} {γ : Type x} {m : Type x → Type
     | some fx => return .done (some fx))
 
 /--
-Steps through the iterator until `f` returns `some` for an element, at which point iteration stops
-and the result of `f` is returned. If the iterator is completely consumed without `f` returning
-`some`, then the result is `none`.
+Returns the first non-`none` result of applying the monadic function `f` to each output
+of the iterator, in order. Returns `none` if `f` returns `none` for all outputs.
+
+`O(|it|)`. Short-circuits when `f` returns `some _`. The outputs of `it` are
+examined in order of iteration.
+
+This function is deprecated. Instead of `it.allowNontermination.findSomeM?`, use `it.findSomeM?`.
+
+Example:
+```lean example
+#eval [7, 6, 5, 8, 1, 2, 6].iter.findSomeM? fun i => do
+  if i < 5 then
+    return some (i * 10)
+  if i ≤ 6 then
+    IO.println s!"Almost! {i}"
+  return none
+```
+```output
+Almost! 6
+Almost! 5
+```
+```output
+some 10
+```
+-/
+@[inline, deprecated Iter.findSomeM? (since := "2025-12-04")]
+def Iter.Partial.findSomeM? {α β : Type w} {γ : Type x} {m : Type x → Type w'} [Monad m]
+    [Iterator α Id β] [IteratorLoop α Id m] (it : Iter.Partial (α := α) β)
+    (f : β → m (Option γ)) :
+    m (Option γ) :=
+  it.it.findSomeM? f
+
+/--
+Returns the first non-`none` result of applying the monadic function `f` to each output
+of the iterator, in order. Returns `none` if `f` returns `none` for all outputs.
+
+`O(|it|)`. Short-circuits when `f` returns `some _`. The outputs of `it` are
+examined in order of iteration.
+
+This variant terminates after finitely many steps and requires a proof that the iterator is
+finite. If such a proof is not available, consider using `Iter.findSomeM?`.
+
+Example:
+```lean example
+#eval [7, 6, 5, 8, 1, 2, 6].iter.findSomeM? fun i => do
+  if i < 5 then
+    return some (i * 10)
+  if i ≤ 6 then
+    IO.println s!"Almost! {i}"
+  return none
+```
+```output
+Almost! 6
+Almost! 5
+```
+```output
+some 10
+```
+-/
+@[inline]
+def Iter.Total.findSomeM? {α β : Type w} {γ : Type x} {m : Type x → Type w'} [Monad m]
+    [Iterator α Id β] [IteratorLoop α Id m] [Finite α Id] (it : Iter.Total (α := α) β)
+    (f : β → m (Option γ)) :
+    m (Option γ) :=
+  it.it.findSomeM? f
+
+/--
+Returns the first non-`none` result of applying `f` to each output of the iterator, in order.
+Returns `none` if `f` returns `none` for all outputs.
+
+`O(|it|)`. Short-circuits when `f` returns `some _`.The outputs of `it` are examined in order of
+iteration.
+
+If the iterator is not finite, this function might run forever. The variant
+`it.ensureTermination.findSome?` always terminates after finitely many steps.
+
+Examples:
+ * `[7, 6, 5, 8, 1, 2, 6].iter.findSome? (fun x => if x < 5 then some (10 * x) else none) = some 10`
+ * `[7, 6, 5, 8, 1, 2, 6].iter.findSome? (fun x => if x < 1 then some (10 * x) else none) = none`
 -/
 @[inline]
 def Iter.findSome? {α β : Type w} {γ : Type x} [Iterator α Id β]
-    [IteratorLoop α Id Id] [Finite α Id] (it : Iter (α := α) β) (f : β → Option γ) :
-    Option γ :=
-  Id.run (it.findSomeM? (pure <| f ·))
-
-@[inline, inherit_doc Iter.findSome?]
-def Iter.Partial.findSome? {α β : Type w} {γ : Type x} [Iterator α Id β]
-    [IteratorLoopPartial α Id Id] (it : Iter.Partial (α := α) β) (f : β → Option γ) :
+    [IteratorLoop α Id Id] (it : Iter (α := α) β) (f : β → Option γ) :
     Option γ :=
   Id.run (it.findSomeM? (pure <| f ·))
 
 /--
-Steps through the iterator until an element satisfies the monadic predicate `f`, at which point
-iteration stops and the element is returned. If no element satisfies `f`, then the result is
-`none`.
+Returns the first non-`none` result of applying `f` to each output of the iterator, in order.
+Returns `none` if `f` returns `none` for all outputs.
+
+`O(|it|)`. Short-circuits when `f` returns `some _`.The outputs of `it` are examined in order of
+iteration.
+
+This function is deprecated. Instead of `it.allowNontermination.findSome?`, use `it.findSome?`.
+
+Examples:
+ * `[7, 6, 5, 8, 1, 2, 6].iter.allowNontermination.findSome? (fun x => if x < 5 then some (10 * x) else none) = some 10`
+ * `[7, 6, 5, 8, 1, 2, 6].iter.allowNontermination.findSome? (fun x => if x < 1 then some (10 * x) else none) = none`
+-/
+@[inline, deprecated Iter.findSome? (since := "2025-12-04")]
+def Iter.Partial.findSome? {α β : Type w} {γ : Type x} [Iterator α Id β]
+    [IteratorLoop α Id Id] (it : Iter.Partial (α := α) β) (f : β → Option γ) :
+    Option γ :=
+  it.it.findSome? f
+
+/--
+Returns the first non-`none` result of applying `f` to each output of the iterator, in order.
+Returns `none` if `f` returns `none` for all outputs.
+
+`O(|it|)`. Short-circuits when `f` returns `some _`.The outputs of `it` are examined in order of
+iteration.
+
+This variant terminates after finitely many steps and requires a proof that the iterator is
+finite. If such a proof is not available, consider using `Iter.findSome?`.
+
+Examples:
+ * `[7, 6, 5, 8, 1, 2, 6].iter.ensureTermination.findSome? (fun x => if x < 5 then some (10 * x) else none) = some 10`
+ * `[7, 6, 5, 8, 1, 2, 6].iter.ensureTermination.findSome? (fun x => if x < 1 then some (10 * x) else none) = none`
+-/
+@[inline]
+def Iter.Total.findSome? {α β : Type w} {γ : Type x} [Iterator α Id β]
+    [IteratorLoop α Id Id] [Finite α Id] (it : Iter.Total (α := α) β) (f : β → Option γ) :
+    Option γ :=
+  it.it.findSome? f
+
+/--
+Returns the first output of the iterator for which the monadic predicate `p` returns `true`, or
+`none` if no such element is found.
+
+`O(|it|)`. Short-circuits when `f` returns `true`. The outputs of `it` are examined in order of
+iteration.
+
+If the iterator is not finite, this function might run forever. The variant
+`it.ensureTermination.findM?` always terminates after finitely many steps.
+
+Example:
+```lean example
+#eval [7, 6, 5, 8, 1, 2, 6].iter.findM? fun i => do
+  if i < 5 then
+    return true
+  if i ≤ 6 then
+    IO.println s!"Almost! {i}"
+  return false
+```
+```output
+Almost! 6
+Almost! 5
+```
+```output
+some 1
+```
 -/
 @[inline]
 def Iter.findM? {α β : Type w} {m : Type w → Type w'} [Monad m] [Iterator α Id β]
-    [IteratorLoop α Id m] [Finite α Id] (it : Iter (α := α) β) (f : β → m (ULift Bool)) :
-    m (Option β) :=
-  it.findSomeM? (fun x => return if (← f x).down then some x else none)
-
-@[inline, inherit_doc Iter.findM?]
-def Iter.Partial.findM? {α β : Type w} {m : Type w → Type w'} [Monad m] [Iterator α Id β]
-    [IteratorLoopPartial α Id m] (it : Iter.Partial (α := α) β) (f : β → m (ULift Bool)) :
+    [IteratorLoop α Id m] (it : Iter (α := α) β) (f : β → m (ULift Bool)) :
     m (Option β) :=
   it.findSomeM? (fun x => return if (← f x).down then some x else none)
 
 /--
-Steps through the iterator until an element satisfies `f`, at which point iteration stops and the
-element is returned. If no element satisfies `f`, then the result is `none`.
+Returns the first output of the iterator for which the monadic predicate `p` returns `true`, or
+`none` if no such element is found.
+
+`O(|it|)`. Short-circuits when `f` returns `true`. The outputs of `it` are examined in order of
+iteration.
+
+This function is deprecated. Instead of `it.ensureTermination.findM?`, use `it.findM?`.
+
+Example:
+```lean example
+#eval [7, 6, 5, 8, 1, 2, 6].iter.findM? fun i => do
+  if i < 5 then
+    return true
+  if i ≤ 6 then
+    IO.println s!"Almost! {i}"
+  return false
+```
+```output
+Almost! 6
+Almost! 5
+```
+```output
+some 1
+```
+-/
+@[inline, deprecated Iter.findM? (since := "2025-12-04")]
+def Iter.Partial.findM? {α β : Type w} {m : Type w → Type w'} [Monad m] [Iterator α Id β]
+    [IteratorLoop α Id m] (it : Iter.Partial (α := α) β) (f : β → m (ULift Bool)) :
+    m (Option β) :=
+  it.it.findM? f
+
+/--
+Returns the first output of the iterator for which the monadic predicate `p` returns `true`, or
+`none` if no such element is found.
+
+`O(|it|)`. Short-circuits when `f` returns `true`. The outputs of `it` are examined in order of
+iteration.
+
+This variant requires terminates after finitely many steps and requires a proof that the iterator is
+finite. If such a proof is not available, consider using `Iter.findM?`.
+
+Example:
+```lean example
+#eval [7, 6, 5, 8, 1, 2, 6].iter.findM? fun i => do
+  if i < 5 then
+    return true
+  if i ≤ 6 then
+    IO.println s!"Almost! {i}"
+  return false
+```
+```output
+Almost! 6
+Almost! 5
+```
+```output
+some 1
+```
+-/
+@[inline]
+def Iter.Total.findM? {α β : Type w} {m : Type w → Type w'} [Monad m] [Iterator α Id β]
+    [IteratorLoop α Id m] [Finite α Id] (it : Iter.Total (α := α) β) (f : β → m (ULift Bool)) :
+    m (Option β) :=
+  it.it.findM? f
+
+/--
+Returns the first output of the iterator for which the predicate `p` returns `true`, or `none` if
+no such output is found.
+
+`O(|it|)`. Short-circuits upon encountering the first match. The elements in `it` are examined in
+order of iteration.
+
+If the iterator is not finite, this function might run forever. The variant
+`it.ensureTermination.find?` always terminates after finitely many steps.
+
+Examples:
+* `[7, 6, 5, 8, 1, 2, 6].iter.find? (· < 5) = some 1`
+* `[7, 6, 5, 8, 1, 2, 6].iter.find? (· < 1) = none`
 -/
 @[inline]
 def Iter.find? {α β : Type w} [Iterator α Id β] [IteratorLoop α Id Id]
-    [Finite α Id] (it : Iter (α := α) β) (f : β → Bool) : Option β :=
+    (it : Iter (α := α) β) (f : β → Bool) : Option β :=
   Id.run (it.findM? (pure <| .up <| f ·))
 
-@[inline, inherit_doc Iter.find?]
-def Iter.Partial.find? {α β : Type w} [Iterator α Id β] [IteratorLoopPartial α Id Id]
+/--
+Returns the first output of the iterator for which the predicate `p` returns `true`, or `none` if
+no such output is found.
+
+`O(|it|)`. Short-circuits upon encountering the first match. The elements in `it` are examined in
+order of iteration.
+
+This function is deprecated. Instead of `it.allowNontermination.find?`, use `it.find?`.
+
+Examples:
+* `[7, 6, 5, 8, 1, 2, 6].iter.allowNontermination.find? (· < 5) = some 1`
+* `[7, 6, 5, 8, 1, 2, 6].iter.allowNontermination.find? (· < 1) = none`
+-/
+@[inline, deprecated Iter.find? (since := "2025-12-04")]
+def Iter.Partial.find? {α β : Type w} [Iterator α Id β] [IteratorLoop α Id Id]
     (it : Iter.Partial (α := α) β) (f : β → Bool) : Option β :=
-  Id.run (it.findM? (pure <| .up <| f ·))
+  it.it.find? f
+
+/--
+Returns the first output of the iterator for which the predicate `p` returns `true`, or `none` if
+no such output is found.
+
+`O(|it|)`. Short-circuits upon encountering the first match. The elements in `it` are examined in
+order of iteration.
+
+This variant terminates after finitely many steps and requires a proof that the iterator is
+finite. If such a proof is not available, consider using `Iter.find?`.
+
+Examples:
+* `[7, 6, 5, 8, 1, 2, 6].iter.find? (· < 5) = some 1`
+* `[7, 6, 5, 8, 1, 2, 6].iter.find? (· < 1) = none`
+-/
+@[inline]
+def Iter.Total.find? {α β : Type w} [Iterator α Id β] [IteratorLoop α Id Id] [Finite α Id]
+    (it : Iter.Total (α := α) β) (f : β → Bool) : Option β :=
+  it.it.find? f
 
 /--
 Steps through the whole iterator, counting the number of outputs emitted.
@@ -287,7 +637,7 @@ Steps through the whole iterator, counting the number of outputs emitted.
 This function's runtime is linear in the number of steps taken by the iterator.
 -/
 @[always_inline, inline, expose]
-def Iter.count {α : Type w} {β : Type w} [Iterator α Id β] [Finite α Id] [IteratorLoop α Id Id]
+def Iter.count {α : Type w} {β : Type w} [Iterator α Id β] [IteratorLoop α Id Id]
     (it : Iter (α := α) β) : Nat :=
   it.toIterM.count.run.down
 
@@ -299,7 +649,7 @@ Steps through the whole iterator, counting the number of outputs emitted.
 This function's runtime is linear in the number of steps taken by the iterator.
 -/
 @[always_inline, inline, expose, deprecated Iter.count (since := "2025-10-29")]
-def Iter.size {α : Type w} {β : Type w} [Iterator α Id β] [Finite α Id] [IteratorLoop α Id Id]
+def Iter.size {α : Type w} {β : Type w} [Iterator α Id β] [IteratorLoop α Id Id]
     (it : Iter (α := α) β) : Nat :=
    it.count
 
@@ -310,10 +660,10 @@ Steps through the whole iterator, counting the number of outputs emitted.
 
 This function's runtime is linear in the number of steps taken by the iterator.
 -/
-@[always_inline, inline, expose]
-def Iter.Partial.count {α : Type w} {β : Type w} [Iterator α Id β] [IteratorLoopPartial α Id Id]
+@[always_inline, inline, expose, deprecated Iter.count (since := "2025-12-04")]
+def Iter.Partial.count {α : Type w} {β : Type w} [Iterator α Id β] [IteratorLoop α Id Id]
     (it : Iter.Partial (α := α) β) : Nat :=
-  it.it.toIterM.allowNontermination.count.run.down
+  it.it.toIterM.count.run.down
 
 /--
 Steps through the whole iterator, counting the number of outputs emitted.
@@ -322,9 +672,9 @@ Steps through the whole iterator, counting the number of outputs emitted.
 
 This function's runtime is linear in the number of steps taken by the iterator.
 -/
-@[always_inline, inline, expose, deprecated Iter.Partial.count (since := "2025-10-29")]
-def Iter.Partial.size {α : Type w} {β : Type w} [Iterator α Id β] [IteratorLoopPartial α Id Id]
+@[always_inline, inline, expose, deprecated Iter.count (since := "2025-10-29")]
+def Iter.Partial.size {α : Type w} {β : Type w} [Iterator α Id β] [IteratorLoop α Id Id]
     (it : Iter.Partial (α := α) β) : Nat :=
-  it.count
+  it.it.count
 
 end Std.Iterators

src/Init/Data/Iterators/Consumers/Monadic/Collect.lean

@@ -7,7 +7,9 @@ module
 
 prelude
 public import Init.Data.Iterators.Consumers.Monadic.Partial
+public import Init.Data.Iterators.Consumers.Monadic.Total
 public import Init.Data.Iterators.Internal.LawfulMonadLiftFunction
+public import Init.WFExtrinsicFix
 
 @[expose] public section
 
@@ -22,9 +24,9 @@ Concretely, the following operations are provided:
 * `IterM.toArray`, collecting the values in an array
 
 Some producers and combinators provide specialized implementations. These are captured by the
-`IteratorCollect` and `IteratorCollectPartial` typeclasses. They should be implemented by all
-types of iterators. A default implementation is provided. The typeclass `LawfulIteratorCollect`
-asserts that an `IteratorCollect` instance equals the default implementation.
+`IteratorCollect` type class. They should be implemented by all types of iterators. A default
+implementation is provided. The typeclass `LawfulIteratorCollect` asserts that an `IteratorCollect`
+instance equals the default implementation.
 -/
 
 namespace Std.Iterators
@@ -51,66 +53,58 @@ class IteratorCollect (α : Type w) (m : Type w → Type w') (n : Type w → Typ
   Maps the emitted values of an iterator using the given function and collects the results in an
   `Array`. This is an internal implementation detail. Consider using `it.map f |>.toArray` instead.
   -/
-  toArrayMapped [Finite α m] :
-    (lift : ⦃δ : Type w⦄ → m δ → n δ) → {γ : Type w} → (β → n γ) → IterM (α := α) m β → n (Array γ)
-
-/--
-`IteratorCollectPartial α m` provides efficient implementations of collectors for `α`-based
-iterators. Right now, it is limited to a potentially optimized partial `toArray` implementation.
-
-This class is experimental and users of the iterator API should not explicitly depend on it.
-They can, however, assume that consumers that require an instance will work for all iterators
-provided by the standard library.
--/
-class IteratorCollectPartial (α : Type w) (m : Type w → Type w') (n : Type w → Type w'')
-    {β : Type w} [Iterator α m β] where
-  /--
-  Maps the emitted values of an iterator using the given function and collects the results in an
-  `Array`. This is an internal implementation detail.
-  Consider using `it.map f |>.allowNontermination.toArray` instead.
-  -/
-  toArrayMappedPartial :
+  toArrayMapped :
     (lift : ⦃δ : Type w⦄ → m δ → n δ) → {γ : Type w} → (β → n γ) → IterM (α := α) m β → n (Array γ)
 
 end Typeclasses
 
 section ToArray
 
+def IterM.DefaultConsumers.toArrayMapped.RecursionRel {α β : Type w} {m : Type w → Type w'}
+    [Iterator α m β] {γ : Type w} (x' x : (_ : IterM (α := α) m β) ×' Array γ) : Prop :=
+  (∃ out, x.1.IsPlausibleStep (.yield x'.1 out) ∧ ∃ fx, x'.2 = x.2.push fx) ∨
+    (x.1.IsPlausibleStep (.skip x'.1) ∧ x'.2 = x.2)
+
 /--
 This is an internal function used in `IteratorCollect.defaultImplementation`.
 
 It iterates over an iterator and applies `f` whenever a value is emitted before inserting the result
 of `f` into an array.
 -/
-@[always_inline, inline]
+@[always_inline, no_expose]
 def IterM.DefaultConsumers.toArrayMapped {α β : Type w} {m : Type w → Type w'}
-    {n : Type w → Type w''} [Monad n] [Iterator α m β] [Finite α m]
+    {n : Type w → Type w''} [Monad n] [Iterator α m β]
     (lift : ⦃α : Type w⦄ → m α → n α) {γ : Type w} (f : β → n γ)
     (it : IterM (α := α) m β) : n (Array γ) :=
+  letI : MonadLift m n := ⟨lift (α := _)⟩
   go it #[]
 where
-  @[specialize]
-  go [Monad n] [Finite α m] (it : IterM (α := α) m β) a := letI : MonadLift m n := ⟨lift (α := _)⟩; do
-    match (← it.step).inflate with
-    | .yield it' b _ => go it' (a.push (← f b))
-    | .skip it' _ => go it' a
-    | .done _ => return a
-  termination_by it.finitelyManySteps
+  @[always_inline]
+  go it (acc : Array γ) : n (Array γ) :=
+    letI : MonadLift m n := ⟨lift (α := _)⟩
+    WellFounded.extrinsicFix₂ (C₂ := fun _ _ => n (Array γ)) (InvImage TerminationMeasures.Finite.Rel (·.1.finitelyManySteps!))
+    (fun (it : IterM (α := α) m β) acc recur => do
+      match (← it.step).inflate with
+      | .yield it' out h =>
+        recur it' (acc.push (← f out)) (by exact TerminationMeasures.Finite.rel_of_yield ‹_›)
+      | .skip it' h => recur it' acc (by exact TerminationMeasures.Finite.rel_of_skip ‹_›)
+      | .done _ => return acc) it acc
 
 /--
-This is the default implementation of the `IteratorLoop` class.
+This is the default implementation of the `IteratorCollect` class.
 It simply iterates through the iterator using `IterM.step`, incrementally building up the desired
 data structure. For certain iterators, more efficient implementations are possible and should be
 used instead.
 -/
-@[always_inline, inline]
+@[always_inline]
 def IteratorCollect.defaultImplementation {α β : Type w} {m : Type w → Type w'}
     {n : Type w → Type w''} [Monad n] [Iterator α m β] :
     IteratorCollect α m n where
   toArrayMapped := IterM.DefaultConsumers.toArrayMapped
 
 /--
-Asserts that a given `IteratorCollect` instance is equal to `IteratorCollect.defaultImplementation`.
+Asserts that a given `IteratorCollect` instance is equal to `IteratorCollect.defaultImplementation`
+*if the underlying iterator is finite*.
 (Even though equal, the given instance might be vastly more efficient.)
 -/
 class LawfulIteratorCollect (α : Type w) (m : Type w → Type w') (n : Type w → Type w'')
@@ -135,62 +129,38 @@ instance (α β : Type w) (m : Type w → Type w') (n : Type w → Type w'') [Mo
   letI : IteratorCollect α m n := .defaultImplementation
   ⟨fun _ => rfl⟩
 
-/--
-This is an internal function used in `IteratorCollectPartial.defaultImplementation`.
-
-It iterates over an iterator and applies `f` whenever a value is emitted before inserting the result
-of `f` into an array.
--/
-@[always_inline, inline]
-partial def IterM.DefaultConsumers.toArrayMappedPartial {α β : Type w} {m : Type w → Type w'}
-    {n : Type w → Type w''} [Monad n] [Iterator α m β]
-    (lift : {α : Type w} → m α → n α) {γ : Type w} (f : β → n γ)
-    (it : IterM (α := α) m β) : n (Array γ) :=
-  go it #[]
-where
-  @[specialize]
-  go [Monad n] (it : IterM (α := α) m β) a := letI : MonadLift m n := ⟨lift⟩; do
-    match (← it.step).inflate with
-    | .yield it' b _ => go it' (a.push (← f b))
-    | .skip it' _ => go it' a
-    | .done _ => return a
-
-/--
-This is the default implementation of the `IteratorLoopPartial` class.
-It simply iterates through the iterator using `IterM.step`, incrementally building up the desired
-data structure. For certain iterators, more efficient implementations are possible and should be
-used instead.
--/
-@[always_inline, inline]
-def IteratorCollectPartial.defaultImplementation {α β : Type w} {m : Type w → Type w'}
-    {n : Type w → Type w''} [Monad n] [Iterator α m β] :
-    IteratorCollectPartial α m n where
-  toArrayMappedPartial := IterM.DefaultConsumers.toArrayMappedPartial
-
 /--
 Traverses the given iterator and stores the emitted values in an array.
 
-This function requires a `Finite` instance proving that the iterator will finish after a finite
-number of steps. If the iterator is not finite or such an instance is not available, consider using
-`it.allowNontermination.toArray` instead of `it.toArray`. However, it is not possible to formally
-verify the behavior of the partial variant.
+If the iterator is not finite, this function might run forever. The variant
+`it.ensureTermination.toArray` always terminates after finitely many steps.
 -/
 @[always_inline, inline]
-def IterM.toArray {α β : Type w} {m : Type w → Type w'} [Monad m]
-    [Iterator α m β] [Finite α m] [IteratorCollect α m m]
-    (it : IterM (α := α) m β) : m (Array β) :=
+def IterM.toArray {α β : Type w} {m : Type w → Type w'} [Monad m] [Iterator α m β]
+    [IteratorCollect α m m] (it : IterM (α := α) m β) : m (Array β) :=
   IteratorCollect.toArrayMapped (fun ⦃_⦄ => id) pure it
 
 /--
 Traverses the given iterator and stores the emitted values in an array.
 
-This is a partial, potentially nonterminating, function. It is not possible to formally verify
-its behavior. If the iterator has a `Finite` instance, consider using `IterM.toArray` instead.
+This function is deprecated. Instead of `it.allowNontermination.toArray`, use `it.toArray`.
+-/
+@[always_inline, inline, deprecated IterM.toArray (since := "2025-10-23")]
+def IterM.Partial.toArray {α : Type w} {m : Type w → Type w'} {β : Type w} [Monad m]
+    [Iterator α m β] (it : IterM.Partial (α := α) m β) [IteratorCollect α m m] : m (Array β) :=
+  it.it.toArray
+
+/--
+Traverses the given iterator and stores the emitted values in an array.
+
+This variant terminates after finitely many steps and requires a proof that the iterator is
+finite. If such a proof is not available, consider using `IterM.toArray`.
 -/
 @[always_inline, inline]
-def IterM.Partial.toArray {α : Type w} {m : Type w → Type w'} {β : Type w} [Monad m]
-    [Iterator α m β] (it : IterM.Partial (α := α) m β) [IteratorCollectPartial α m m] : m (Array β) :=
-  IteratorCollectPartial.toArrayMappedPartial (fun ⦃_⦄ => id) pure it.it
+def IterM.Total.toArray {α : Type w} {m : Type w → Type w'} {β : Type w} [Monad m]
+    [Iterator α m β] [Finite α m] (it : IterM.Total (α := α) m β) [IteratorCollect α m m] :
+    m (Array β) :=
+  it.it.toArray
 
 end ToArray
 
@@ -198,67 +168,82 @@ end ToArray
 Traverses the given iterator and stores the emitted values in reverse order in a list. Because
 lists are prepend-only, this `toListRev` is usually more efficient that `toList`.
 
-This function requires a `Finite` instance proving that the iterator will finish after a finite
-number of steps. If the iterator is not finite or such an instance is not available, consider using
-`it.allowNontermination.toListRev` instead of `it.toListRev`. However, it is not possible to
-formally verify the behavior of the partial variant.
+If the iterator is not finite, this function might run forever. The variant
+`it.ensureTermination.toListRev` always terminates after finitely many steps.
 -/
-@[inline]
+@[always_inline, inline]
 def IterM.toListRev {α : Type w} {m : Type w → Type w'} [Monad m] {β : Type w}
-    [Iterator α m β] [Finite α m] (it : IterM (α := α) m β) : m (List β) :=
+    [Iterator α m β] (it : IterM (α := α) m β) : m (List β) :=
   go it []
 where
-  go [Finite α m] it bs := do
-    match (← it.step).inflate with
-    | .yield it' b _ => go it' (b :: bs)
-    | .skip it' _ => go it' bs
-    | .done _ => return bs
-  termination_by it.finitelyManySteps
+  @[always_inline, inline]
+  go (it : IterM m β) acc :=
+    WellFounded.extrinsicFix₂ (InvImage TerminationMeasures.Finite.Rel (·.1.finitelyManySteps!))
+      (fun it acc recur => do
+        match (← it.step).inflate with
+        | .yield it' out h => recur it' (out :: acc) (TerminationMeasures.Finite.rel_of_yield h)
+        | .skip it' h => recur it' acc (TerminationMeasures.Finite.rel_of_skip h)
+        | .done _ => return acc) it acc
 
 /--
 Traverses the given iterator and stores the emitted values in reverse order in a list. Because
 lists are prepend-only, this `toListRev` is usually more efficient that `toList`.
 
-This is a partial, potentially nonterminating, function. It is not possible to formally verify
-its behavior. If the iterator has a `Finite` instance, consider using `IterM.toListRev` instead.
+This function is deprecated. Instead of `it.allowNontermination.toListRev`, use `it.toListRev`.
 -/
-@[always_inline, inline]
+@[always_inline, inline, deprecated IterM.toListRev (since := "2025-10-23")]
 partial def IterM.Partial.toListRev {α : Type w} {m : Type w → Type w'} [Monad m] {β : Type w}
     [Iterator α m β] (it : IterM.Partial (α := α) m β) : m (List β) :=
-  go it.it []
-where
-  @[specialize]
-  go it bs := do
-    match (← it.step).inflate with
-    | .yield it' b _ => go it' (b :: bs)
-    | .skip it' _ => go it' bs
-    | .done _ => return bs
+  it.it.toListRev
+
+/--
+Traverses the given iterator and stores the emitted values in reverse order in a list. Because
+lists are prepend-only, this `toListRev` is usually more efficient that `toList`.
+
+This variant terminates after finitely many steps and requires a proof that the iterator is
+finite. If such a proof is not available, consider using `IterM.toListRev`.
+-/
+@[always_inline, inline]
+def IterM.Total.toListRev {α : Type w} {m : Type w → Type w'} {β : Type w} [Monad m]
+    [Iterator α m β] [Finite α m] (it : IterM.Total (α := α) m β) :
+    m (List β) :=
+  it.it.toListRev
 
 /--
 Traverses the given iterator and stores the emitted values in a list. Because
 lists are prepend-only, `toListRev` is usually more efficient that `toList`.
 
-This function requires a `Finite` instance proving that the iterator will finish after a finite
-number of steps. If the iterator is not finite or such an instance is not available, consider using
-`it.allowNontermination.toList` instead of `it.toList`. However, it is not possible to
-formally verify the behavior of the partial variant.
+If the iterator is not finite, this function might run forever. The variant
+`it.ensureTermination.toList` always terminates after finitely many steps.
 -/
 @[always_inline, inline]
 def IterM.toList {α : Type w} {m : Type w → Type w'} [Monad m] {β : Type w}
-    [Iterator α m β] [Finite α m] [IteratorCollect α m m] (it : IterM (α := α) m β) : m (List β) :=
+    [Iterator α m β] [IteratorCollect α m m] (it : IterM (α := α) m β) : m (List β) :=
   Array.toList <$> IterM.toArray it
 
 /--
 Traverses the given iterator and stores the emitted values in a list. Because
 lists are prepend-only, `toListRev` is usually more efficient that `toList`.
 
-This is a partial, potentially nonterminating, function. It is not possible to formally verify
-its behavior. If the iterator has a `Finite` instance, consider using `IterM.toList` instead.
+This function is deprecated. Instead of `it.allowNontermination.toList`, use `it.toList`.
+-/
+@[always_inline, inline, deprecated IterM.toList (since := "2025-10-23")]
+def IterM.Partial.toList {α : Type w} {m : Type w → Type w'} [Monad m] {β : Type w}
+    [Iterator α m β] (it : IterM.Partial (α := α) m β) [IteratorCollect α m m] :
+    m (List β) :=
+  Array.toList <$> it.it.toArray
+
+/--
+Traverses the given iterator and stores the emitted values in a list. Because
+lists are prepend-only, `toListRev` is usually more efficient that `toList`.
+
+This variant terminates after finitely many steps and requires a proof that the iterator is
+finite. If such a proof is not available, consider using `IterM.toList`.
 -/
 @[always_inline, inline]
-def IterM.Partial.toList {α : Type w} {m : Type w → Type w'} [Monad m] {β : Type w}
-    [Iterator α m β] (it : IterM.Partial (α := α) m β) [IteratorCollectPartial α m m] :
+def IterM.Total.toList {α : Type w} {m : Type w → Type w'} {β : Type w} [Monad m]
+    [Iterator α m β] [Finite α m] (it : IterM.Total (α := α) m β) [IteratorCollect α m m] :
     m (List β) :=
-  Array.toList <$> it.toArray
+  it.it.toList
 
 end Std.Iterators

src/Init/Data/Iterators/Consumers/Monadic/Total.lean

@@ -0,0 +1,36 @@
+/-
+Copyright (c) 2025 Lean FRO, LLC. All rights reserved.
+Released under Apache 2.0 license as described in the file LICENSE.
+Authors: Paul Reichert
+-/
+module
+
+prelude
+public import Init.Data.Iterators.Basic
+
+set_option doc.verso true
+
+public section
+
+namespace Std.Iterators
+
+structure IterM.Total {α : Type w} (m : Type w → Type w') (β : Type w) where
+  it : IterM (α := α) m β
+
+/--
+For an iterator {name}`it`, {lean}`it.ensureTermination` provides variants of consumers that always
+terminate.
+-/
+@[always_inline, inline]
+def IterM.ensureTermination {α : Type w} {β : Type w} {m : Type w → Type w'}
+    (it : IterM (α := α) m β) :
+  IterM.Total (α := α) m β :=
+  ⟨it⟩
+
+/--
+A wrapper around an iterator that provides strictly terminating consumers. See
+{name}`IterM.ensureTermination`.
+-/
+add_decl_doc IterM.Total
+
+end Std.Iterators

src/Init/Data/Iterators/Consumers/Total.lean

@@ -0,0 +1,36 @@
+/-
+Copyright (c) 2025 Lean FRO, LLC. All rights reserved.
+Released under Apache 2.0 license as described in the file LICENSE.
+Authors: Paul Reichert
+-/
+module
+
+prelude
+public import Init.Data.Iterators.Basic
+
+set_option doc.verso true
+
+public section
+
+namespace Std.Iterators
+
+structure Iter.Total {α : Type w} (β : Type w) where
+  it : Iter (α := α) β
+
+/--
+For an iterator {name}`it`, {lean}`it.ensureTermination` provides variants of consumers that always
+terminate.
+-/
+@[always_inline, inline]
+def Iter.ensureTermination {α : Type w} {β : Type w}
+    (it : Iter (α := α) β) :
+  Iter.Total (α := α) β :=
+  ⟨it⟩
+
+/--
+A wrapper around an iterator that provides strictly terminating consumers. See
+{name}`Iter.ensureTermination`.
+-/
+add_decl_doc Iter.Total
+
+end Std.Iterators

src/Init/Data/Iterators/Lemmas/Combinators/Monadic/FilterMap.lean

@@ -247,9 +247,8 @@ instance {α β γ : Type w} {m : Type w → Type w'} {n : Type w → Type w''}
     have : it = IterM.mapWithPostcondition _ it.internalState.inner := by rfl
     generalize it.internalState.inner = it at *
     cases this
-    simp only [LawfulIteratorCollect.toArrayMapped_eq]
     simp only [IteratorCollect.toArrayMapped]
-    rw [LawfulIteratorCollect.toArrayMapped_eq]
+    simp only [LawfulIteratorCollect.toArrayMapped_eq]
     induction it using IterM.inductSteps with | step it ih_yield ih_skip
     rw [IterM.DefaultConsumers.toArrayMapped_eq_match_step]
     rw [IterM.DefaultConsumers.toArrayMapped_eq_match_step]
@@ -487,13 +486,8 @@ theorem IterM.toList_map {α β β' : Type w} {m : Type w → Type w'} [Monad m]
   · simp [instIteratorMap, inferInstanceAs]
     congr
     simp
-  · refine heq_of_eqRec_eq ?_ rfl
-    congr
+  · congr
     simp only [Map, PostconditionT.map_pure, Function.comp_apply]
-    simp only [instIteratorMap, inferInstanceAs, Function.comp_apply]
-    congr
-    simp
-  · simp [Map]
   · simp only [instIteratorMap, inferInstanceAs, Function.comp_apply]
     congr
     simp

src/Init/Data/Iterators/Lemmas/Consumers/Collect.lean

@@ -11,6 +11,8 @@ public import Init.Data.Iterators.Lemmas.Consumers.Monadic.Collect
 public import Init.Data.Iterators.Consumers.Access
 import all Init.Data.Iterators.Consumers.Access
 import all Init.Data.Iterators.Consumers.Collect
+import all Init.Data.Iterators.Consumers.Total
+import all Init.Data.Iterators.Consumers.Monadic.Total
 
 public section
 
@@ -31,6 +33,23 @@ theorem Iter.toListRev_eq_toListRev_toIterM {α β} [Iterator α Id β] [Finite
     it.toListRev = it.toIterM.toListRev.run :=
   (rfl)
 
+@[simp]
+theorem Iter.toArray_ensureTermination {α β} [Iterator α Id β] [Finite α Id] [IteratorCollect α Id Id]
+    [LawfulIteratorCollect α Id Id] {it : Iter (α := α) β} :
+    it.ensureTermination.toArray = it.toArray :=
+  (rfl)
+
+@[simp]
+theorem Iter.toList_ensureTermination {α β} [Iterator α Id β] [Finite α Id] [IteratorCollect α Id Id]
+    [LawfulIteratorCollect α Id Id] {it : Iter (α := α) β} :
+    it.ensureTermination.toList = it.toList :=
+  (rfl)
+
+@[simp]
+theorem Iter.toListRev_ensureTermination_eq_toListRev {α β} [Iterator α Id β] [Finite α Id]
+    {it : Iter (α := α) β} : it.ensureTermination.toListRev = it.toListRev :=
+  (rfl)
+
 @[simp]
 theorem IterM.toList_toIter {α β} [Iterator α Id β] [Finite α Id] [IteratorCollect α Id Id]
     {it : IterM (α := α) Id β} :
@@ -49,12 +68,22 @@ theorem Iter.toList_toArray {α β} [Iterator α Id β] [Finite α Id] [Iterator
     it.toArray.toList = it.toList := by
   simp [toArray_eq_toArray_toIterM, toList_eq_toList_toIterM, ← IterM.toList_toArray]
 
+theorem Iter.toList_toArray_ensureTermination {α β} [Iterator α Id β] [Finite α Id]
+    [IteratorCollect α Id Id] [LawfulIteratorCollect α Id Id] {it : Iter (α := α) β} :
+    it.ensureTermination.toArray.toList = it.toList := by
+  simp
+
 @[simp]
 theorem Iter.toArray_toList {α β} [Iterator α Id β] [Finite α Id] [IteratorCollect α Id Id]
     [LawfulIteratorCollect α Id Id] {it : Iter (α := α) β} :
     it.toList.toArray = it.toArray := by
   simp [toArray_eq_toArray_toIterM, toList_eq_toList_toIterM, ← IterM.toArray_toList]
 
+theorem Iter.toArray_toList_ensureTermination {α β} [Iterator α Id β] [Finite α Id]
+    [IteratorCollect α Id Id] [LawfulIteratorCollect α Id Id] {it : Iter (α := α) β} :
+    it.ensureTermination.toList.toArray = it.toArray := by
+  simp
+
 @[simp]
 theorem Iter.reverse_toListRev [Iterator α Id β] [Finite α Id]
     [IteratorCollect α Id Id] [LawfulIteratorCollect α Id Id]
@@ -62,11 +91,21 @@ theorem Iter.reverse_toListRev [Iterator α Id β] [Finite α Id]
     it.toListRev.reverse = it.toList := by
   simp [toListRev_eq_toListRev_toIterM, toList_eq_toList_toIterM, ← IterM.reverse_toListRev]
 
+theorem Iter.reverse_toListRev_ensureTermination [Iterator α Id β] [Finite α Id]
+    [IteratorCollect α Id Id] [LawfulIteratorCollect α Id Id] {it : Iter (α := α) β} :
+    it.ensureTermination.toListRev.reverse = it.toList := by
+  simp
+
 theorem Iter.toListRev_eq {α β} [Iterator α Id β] [Finite α Id] [IteratorCollect α Id Id]
     [LawfulIteratorCollect α Id Id] {it : Iter (α := α) β} :
     it.toListRev = it.toList.reverse := by
   simp [Iter.toListRev_eq_toListRev_toIterM, Iter.toList_eq_toList_toIterM, IterM.toListRev_eq]
 
+theorem Iter.toListRev_ensureTermination {α β} [Iterator α Id β] [Finite α Id]
+    [IteratorCollect α Id Id] [LawfulIteratorCollect α Id Id] {it : Iter (α := α) β} :
+    it.ensureTermination.toListRev = it.toList.reverse := by
+  simp [toListRev_eq]
+
 theorem Iter.toArray_eq_match_step {α β} [Iterator α Id β] [Finite α Id] [IteratorCollect α Id Id]
     [LawfulIteratorCollect α Id Id] {it : Iter (α := α) β} :
     it.toArray = match it.step.val with
@@ -78,6 +117,14 @@ theorem Iter.toArray_eq_match_step {α β} [Iterator α Id β] [Finite α Id] [I
   generalize it.toIterM.step.run = step
   cases step.inflate using PlausibleIterStep.casesOn <;> simp
 
+theorem Iter.toArray_ensureTermination_eq_match_step {α β} [Iterator α Id β] [Finite α Id] [IteratorCollect α Id Id]
+    [LawfulIteratorCollect α Id Id] {it : Iter (α := α) β} :
+    it.ensureTermination.toArray = match it.step.val with
+      | .yield it' out => #[out] ++ it'.toArray
+      | .skip it' => it'.toArray
+      | .done => #[] := by
+  rw [toArray_ensureTermination, toArray_eq_match_step]
+
 theorem Iter.toList_eq_match_step {α β} [Iterator α Id β] [Finite α Id] [IteratorCollect α Id Id]
     [LawfulIteratorCollect α Id Id] {it : Iter (α := α) β} :
     it.toList = match it.step.val with
@@ -87,6 +134,14 @@ theorem Iter.toList_eq_match_step {α β} [Iterator α Id β] [Finite α Id] [It
   rw [← Iter.toList_toArray, Iter.toArray_eq_match_step]
   split <;> simp [Iter.toList_toArray]
 
+theorem Iter.toList_ensureTermination_eq_match_step {α β} [Iterator α Id β] [Finite α Id] [IteratorCollect α Id Id]
+    [LawfulIteratorCollect α Id Id] {it : Iter (α := α) β} :
+    it.ensureTermination.toList = match it.step.val with
+      | .yield it' out => out :: it'.toList
+      | .skip it' => it'.toList
+      | .done => [] := by
+  rw [toList_ensureTermination, toList_eq_match_step]
+
 theorem Iter.toListRev_eq_match_step {α β} [Iterator α Id β] [Finite α Id] {it : Iter (α := α) β} :
     it.toListRev = match it.step.val with
       | .yield it' out => it'.toListRev ++ [out]
@@ -96,6 +151,13 @@ theorem Iter.toListRev_eq_match_step {α β} [Iterator α Id β] [Finite α Id]
   generalize it.toIterM.step.run = step
   cases step.inflate using PlausibleIterStep.casesOn <;> simp
 
+theorem Iter.toListRev_ensureTermination_eq_match_step {α β} [Iterator α Id β] [Finite α Id] {it : Iter (α := α) β} :
+    it.ensureTermination.toListRev = match it.step.val with
+      | .yield it' out => it'.toListRev ++ [out]
+      | .skip it' => it'.toListRev
+      | .done => [] := by
+  rw [toListRev_ensureTermination_eq_toListRev, toListRev_eq_match_step]
+
 theorem Iter.getElem?_toList_eq_atIdxSlow? {α β}
     [Iterator α Id β] [Finite α Id] [IteratorCollect α Id Id] [LawfulIteratorCollect α Id Id]
     {it : Iter (α := α) β} {k : Nat} :

src/Init/Data/Iterators/Lemmas/Consumers/Loop.lean

@@ -23,22 +23,22 @@ theorem Iter.forIn'_eq {α β : Type w} [Iterator α Id β] [Finite α Id]
     {f : (b : β) → it.IsPlausibleIndirectOutput b → γ → m (ForInStep γ)} :
     letI : ForIn' m (Iter (α := α) β) β _ := Iter.instForIn'
     ForIn'.forIn' it init f =
-      IterM.DefaultConsumers.forIn' (fun _ _ f x => f x.run) γ (fun _ _ _ => True)
-        IteratorLoop.wellFounded_of_finite it.toIterM init _ (fun _ => id)
-          (fun out h acc => (⟨·, .intro⟩) <$>
-            f out (Iter.isPlausibleIndirectOutput_iff_isPlausibleIndirectOutput_toIterM.mpr h) acc) := by
-  simp [instForIn', ForIn'.forIn', IteratorLoop.finiteForIn', hl.lawful (fun γ δ f x => f x.run),
-    IteratorLoop.defaultImplementation]
+      IterM.DefaultConsumers.forIn' (n := m) (fun _ _ f x => f x.run) γ (fun _ _ _ => True)
+        it.toIterM init _ (fun _ => id)
+          (fun out h acc => return ⟨← f out (Iter.isPlausibleIndirectOutput_iff_isPlausibleIndirectOutput_toIterM.mpr h) acc, trivial⟩) := by
+  simp only [instForIn', ForIn'.forIn', IteratorLoop.finiteForIn']
+  have : ∀ a b c, f a b c = (Subtype.val <$> (⟨·, trivial⟩) <$> f a b c) := by simp
+  simp +singlePass only [this]
+  rw [hl.lawful (fun _ _ f x => f x.run) (wf := IteratorLoop.wellFounded_of_finite)]
+  simp [IteratorLoop.defaultImplementation]
 
 theorem Iter.forIn_eq {α β : Type w} [Iterator α Id β] [Finite α Id]
     {m : Type x → Type x'} [Monad m] [LawfulMonad m] [IteratorLoop α Id m]
     [hl : LawfulIteratorLoop α Id m] {γ : Type x} {it : Iter (α := α) β} {init : γ}
     {f : (b : β) → γ → m (ForInStep γ)} :
     ForIn.forIn it init f =
-      IterM.DefaultConsumers.forIn' (fun _ _ f c => f c.run) γ (fun _ _ _ => True)
-        IteratorLoop.wellFounded_of_finite it.toIterM init _ (fun _ => id)
-          (fun out _ acc => (⟨·, .intro⟩) <$>
-            f out acc) := by
+      IterM.DefaultConsumers.forIn' (n := m) (fun _ _ f c => f c.run) γ (fun _ _ _ => True)
+        it.toIterM init _ (fun _ => id) (fun out _ acc => return ⟨← f out acc, trivial⟩) := by
   simp [ForIn.forIn, forIn'_eq, -forIn'_eq_forIn]
 
 @[congr] theorem Iter.forIn'_congr {α β : Type w} {m : Type w → Type w'} [Monad m]
@@ -106,20 +106,24 @@ theorem Iter.forIn'_eq_match_step {α β : Type w} [Iterator α Id β]
             fun out h' acc => f out (.indirect ⟨_, rfl, h⟩ h') acc
       | .done _ => return init) := by
   simp only [forIn'_eq]
-  rw [IterM.DefaultConsumers.forIn'_eq_match_step]
-  simp only [bind_map_left, Iter.step]
+  rw [IterM.DefaultConsumers.forIn'_eq_match_step (fun _ _ _ => True)
+    IteratorLoop.wellFounded_of_finite]
+  simp only [Iter.step]
   cases it.toIterM.step.run.inflate using PlausibleIterStep.casesOn
-  · simp only [IterM.Step.toPure_yield, PlausibleIterStep.yield, toIter_toIterM, toIterM_toIter]
+  · simp only [IterM.Step.toPure_yield, PlausibleIterStep.yield, toIter_toIterM, toIterM_toIter,
+      bind_assoc]
     apply bind_congr
     intro forInStep
     cases forInStep
     · simp
-    · simp only
-      apply IterM.DefaultConsumers.forIn'_eq_forIn'
-      intros; congr
+    · simp only [pure_bind]
+      apply IterM.DefaultConsumers.forIn'_eq_forIn' (fun _ _ _ => True)
+        IteratorLoop.wellFounded_of_finite
+      · simp
   · simp only
-    apply IterM.DefaultConsumers.forIn'_eq_forIn'
-    intros; congr
+    apply IterM.DefaultConsumers.forIn'_eq_forIn' (fun _ _ _ => True)
+      IteratorLoop.wellFounded_of_finite
+    · simp
   · simp
 
 theorem Iter.forIn_eq_match_step {α β : Type w} [Iterator α Id β]

src/Init/Data/Iterators/Lemmas/Consumers/Monadic/Collect.lean

@@ -10,29 +10,49 @@ public import Init.Data.Array.Lemmas
 public import Init.Data.Iterators.Lemmas.Monadic.Basic
 public import Init.Data.Iterators.Consumers.Monadic.Collect
 import all Init.Data.Iterators.Consumers.Monadic.Collect
+import all Init.Data.Iterators.Consumers.Monadic.Total
+import all Init.WFExtrinsicFix
 
 public section
 
 namespace Std.Iterators
+open Std.Internal
 
 variable {α β γ : Type w} {m : Type w → Type w'} {n : Type w → Type w''}
   {lift : ⦃δ : Type w⦄ → m δ → n δ} {f : β → n γ} {it : IterM (α := α) m β}
 
-theorem IterM.DefaultConsumers.toArrayMapped.go.aux₁ [Monad n] [LawfulMonad n] [Iterator α m β]
-    [Finite α m] {b : γ} {bs : Array γ} :
+private theorem IterM.DefaultConsumers.toArrayMapped.go_eq [Monad n]
+    [Iterator α m β] [LawfulMonad n] [Finite α m] {acc : Array γ} :
+    letI : MonadLift m n := ⟨lift (δ := _)⟩
+    go lift f it acc (m := m) = (do
+      match (← it.step).inflate.val with
+      | .yield it' out => go lift f it' (acc.push (← f out))
+      | .skip it' => go lift f it' acc
+      | .done => return acc) := by
+  letI : MonadLift m n := ⟨lift (δ := _)⟩
+  rw [toArrayMapped.go, WellFounded.extrinsicFix₂_eq_apply]
+  · simp only
+    apply bind_congr; intro step
+    cases step.inflate using PlausibleIterStep.casesOn
+    · apply bind_congr; intro fx
+      simp [go]
+    · simp [go]
+    · simp
+  · simp only [show (IterM.finitelyManySteps! = IterM.finitelyManySteps) by rfl]
+    apply InvImage.wf
+    exact WellFoundedRelation.wf
+
+private theorem IterM.DefaultConsumers.toArrayMapped.go.aux₁ [Monad n] [LawfulMonad n]
+    [Iterator α m β] [Finite α m] {b : γ} {bs : Array γ} :
     IterM.DefaultConsumers.toArrayMapped.go lift f it (#[b] ++ bs) (m := m) =
       (#[b] ++ ·) <$> IterM.DefaultConsumers.toArrayMapped.go lift f it bs (m := m) := by
-  induction it, bs using IterM.DefaultConsumers.toArrayMapped.go.induct with | _ it bs ih₁ ih₂
-  rw [go, map_eq_pure_bind, go, bind_assoc]
-  apply bind_congr
-  intro step
-  split
-  · simp [ih₁ _ _ ‹_›]
-  · simp [ih₂ _ ‹_›]
-  · simp
+  induction it using IterM.inductSteps generalizing bs with | step it ihy ihs
+  rw [go_eq, map_eq_pure_bind, go_eq, bind_assoc]
+  apply bind_congr; intro step
+  cases step.inflate using PlausibleIterStep.casesOn <;> simp (discharger := assumption) [ihy, ihs]
 
-theorem IterM.DefaultConsumers.toArrayMapped.go.aux₂ [Monad n] [LawfulMonad n] [Iterator α m β]
-    [Finite α m] {acc : Array γ} :
+private theorem IterM.DefaultConsumers.toArrayMapped.go.aux₂ [Monad n] [LawfulMonad n]
+    [Iterator α m β] [Finite α m] {acc : Array γ} :
     IterM.DefaultConsumers.toArrayMapped.go lift f it acc (m := m) =
       (acc ++ ·) <$> IterM.DefaultConsumers.toArrayMapped lift f it (m := m) := by
   rw [← Array.toArray_toList (xs := acc)]
@@ -51,12 +71,18 @@ theorem IterM.DefaultConsumers.toArrayMapped_eq_match_step [Monad n] [LawfulMona
         return #[← f out] ++ (← IterM.DefaultConsumers.toArrayMapped lift f it' (m := m))
       | .skip it' => IterM.DefaultConsumers.toArrayMapped lift f it' (m := m)
       | .done => return #[]) := by
-  rw [IterM.DefaultConsumers.toArrayMapped, IterM.DefaultConsumers.toArrayMapped.go]
+  rw [IterM.DefaultConsumers.toArrayMapped, IterM.DefaultConsumers.toArrayMapped.go_eq]
   apply bind_congr
   intro step
   cases step.inflate using PlausibleIterStep.casesOn <;>
     simp [IterM.DefaultConsumers.toArrayMapped.go.aux₂]
 
+@[simp]
+theorem IterM.toArray_ensureTermination [Monad m] [Iterator α m β] [Finite α m]
+    [IteratorCollect α m m] {it : IterM (α := α) m β} :
+    it.ensureTermination.toArray = it.toArray :=
+  (rfl)
+
 theorem IterM.toArray_eq_match_step [Monad m] [LawfulMonad m] [Iterator α m β] [Finite α m]
     [IteratorCollect α m m] [LawfulIteratorCollect α m m] :
     it.toArray = (do
@@ -68,18 +94,43 @@ theorem IterM.toArray_eq_match_step [Monad m] [LawfulMonad m] [Iterator α m β]
   rw [IterM.DefaultConsumers.toArrayMapped_eq_match_step]
   simp [bind_pure_comp, pure_bind]
 
+theorem IterM.toArray_ensureTermination_eq_match_step [Monad m] [LawfulMonad m] [Iterator α m β]
+    [Finite α m] [IteratorCollect α m m] [LawfulIteratorCollect α m m] :
+    it.ensureTermination.toArray = (do
+      match (← it.step).inflate.val with
+      | .yield it' out => return #[out] ++ (← it'.toArray)
+      | .skip it' => it'.toArray
+      | .done => return #[]) := by
+  rw [toArray_ensureTermination, toArray_eq_match_step]
+
+@[simp]
+theorem IterM.toList_ensureTermination [Monad m] [Iterator α m β] [Finite α m]
+    [IteratorCollect α m m] {it : IterM (α := α) m β} :
+    it.ensureTermination.toList = it.toList :=
+  (rfl)
+
 @[simp]
 theorem IterM.toList_toArray [Monad m] [Iterator α m β] [Finite α m] [IteratorCollect α m m]
     {it : IterM (α := α) m β} :
     Array.toList <$> it.toArray = it.toList := by
   simp [IterM.toList]
 
+theorem IterM.toList_toArray_ensureTermination [Monad m] [Iterator α m β] [Finite α m]
+    [IteratorCollect α m m] {it : IterM (α := α) m β} :
+    Array.toList <$> it.ensureTermination.toArray = it.toList := by
+  simp
+
 @[simp]
 theorem IterM.toArray_toList [Monad m] [LawfulMonad m] [Iterator α m β] [Finite α m]
     [IteratorCollect α m m] {it : IterM (α := α) m β} :
     List.toArray <$> it.toList = it.toArray := by
   simp [IterM.toList, -toList_toArray]
 
+theorem IterM.toArray_toList_ensureTermination [Monad m] [LawfulMonad m] [Iterator α m β] [Finite α m]
+    [IteratorCollect α m m] {it : IterM (α := α) m β} :
+    List.toArray <$> it.ensureTermination.toList = it.toArray := by
+  rw [toList_ensureTermination, toArray_toList]
+
 theorem IterM.toList_eq_match_step [Monad m] [LawfulMonad m] [Iterator α m β] [Finite α m]
     [IteratorCollect α m m] [LawfulIteratorCollect α m m] {it : IterM (α := α) m β} :
     it.toList = (do
@@ -93,17 +144,49 @@ theorem IterM.toList_eq_match_step [Monad m] [LawfulMonad m] [Iterator α m β]
   intro step
   split <;> simp
 
-theorem IterM.toListRev.go.aux₁ [Monad m] [LawfulMonad m] [Iterator α m β] [Finite α m]
+theorem IterM.toList_ensureTermination_eq_match_step [Monad m] [LawfulMonad m] [Iterator α m β]
+    [Finite α m] [IteratorCollect α m m] [LawfulIteratorCollect α m m] {it : IterM (α := α) m β} :
+    it.ensureTermination.toList = (do
+      match (← it.step).inflate.val with
+      | .yield it' out => return out :: (← it'.toList)
+      | .skip it' => it'.toList
+      | .done => return []) := by
+  rw [toList_ensureTermination, toList_eq_match_step]
+
+@[simp]
+theorem IterM.toListRev_ensureTermination_eq_toListRev [Monad m] [Iterator α m β] [Finite α m]
+    {it : IterM (α := α) m β} :
+    it.ensureTermination.toListRev = it.toListRev :=
+  (rfl)
+
+private theorem IterM.toListRev.go_eq [Monad m] [LawfulMonad m] [Iterator α m β] [Finite α m]
+    {it : IterM (α := α) m β} {bs : List β} :
+    go it bs = (do
+      match (← it.step).inflate.val with
+      | .yield it' out => go it' (out :: bs)
+      | .skip it' => go it' bs
+      | .done => return bs) := by
+  rw [go, WellFounded.extrinsicFix₂_eq_apply]
+  · apply bind_congr; intro step
+    cases step.inflate using PlausibleIterStep.casesOn <;> simp [go]
+  · simp only [show (IterM.finitelyManySteps! = IterM.finitelyManySteps) by rfl]
+    apply InvImage.wf
+    exact WellFoundedRelation.wf
+
+private theorem IterM.toListRev.go.aux₁ [Monad m] [LawfulMonad m] [Iterator α m β] [Finite α m]
     {it : IterM (α := α) m β} {b : β} {bs : List β} :
     IterM.toListRev.go it (bs ++ [b]) = (· ++ [b]) <$> IterM.toListRev.go it bs:= by
-  induction it, bs using IterM.toListRev.go.induct with | _ it bs ih₁ ih₂
-  rw [go, go, map_eq_pure_bind, bind_assoc]
+  induction it using IterM.inductSteps generalizing bs with | step it ihy ihs
+  rw [go_eq, go_eq, map_eq_pure_bind, bind_assoc]
   apply bind_congr
   intro step
-  simp only [List.cons_append] at ih₁
-  split <;> simp [*]
+  cases step.inflate using PlausibleIterStep.casesOn
+  · simpa using ihy ‹_› (bs := _ :: bs)
+  · simpa using ihs ‹_›
+  · simp
 
-theorem IterM.toListRev.go.aux₂ [Monad m] [LawfulMonad m] [Iterator α m β] [Finite α m]
+
+private theorem IterM.toListRev.go.aux₂ [Monad m] [LawfulMonad m] [Iterator α m β] [Finite α m]
     {it : IterM (α := α) m β} {acc : List β} :
     IterM.toListRev.go it acc = (· ++ acc) <$> it.toListRev := by
   rw [← List.reverse_reverse (as := acc)]
@@ -120,11 +203,21 @@ theorem IterM.toListRev_eq_match_step [Monad m] [LawfulMonad m] [Iterator α m
       | .skip it' => it'.toListRev
       | .done => return []) := by
   simp [IterM.toListRev]
-  rw [toListRev.go]
+  rw [toListRev.go_eq]
   apply bind_congr
   intro step
   cases step.inflate using PlausibleIterStep.casesOn <;> simp [IterM.toListRev.go.aux₂]
 
+theorem IterM.toListRev_ensureTermination_eq_match_step [Monad m] [LawfulMonad m] [Iterator α m β]
+    [Finite α m] {it : IterM (α := α) m β} :
+    it.ensureTermination.toListRev = (do
+      match (← it.step).inflate.val with
+      | .yield it' out => return (← it'.toListRev) ++ [out]
+      | .skip it' => it'.toListRev
+      | .done => return []) := by
+  rw [toListRev_ensureTermination_eq_toListRev, toListRev_eq_match_step]
+
+@[simp]
 theorem IterM.reverse_toListRev [Monad m] [LawfulMonad m] [Iterator α m β] [Finite α m]
     [IteratorCollect α m m] [LawfulIteratorCollect α m m]
     {it : IterM (α := α) m β} :
@@ -137,19 +230,31 @@ theorem IterM.reverse_toListRev [Monad m] [LawfulMonad m] [Iterator α m β] [Fi
   intro step
   cases step.inflate using PlausibleIterStep.casesOn <;> simp (discharger := assumption) [ihy, ihs]
 
+@[simp]
+theorem IterM.reverse_toListRev_ensureTermination [Monad m] [LawfulMonad m] [Iterator α m β]
+    [Finite α m] [IteratorCollect α m m] [LawfulIteratorCollect α m m]
+    {it : IterM (α := α) m β} :
+    List.reverse <$> it.ensureTermination.toListRev = it.toList := by
+  rw [toListRev_ensureTermination_eq_toListRev, reverse_toListRev]
+
 theorem IterM.toListRev_eq [Monad m] [LawfulMonad m] [Iterator α m β] [Finite α m]
     [IteratorCollect α m m] [LawfulIteratorCollect α m m]
     {it : IterM (α := α) m β} :
     it.toListRev = List.reverse <$> it.toList := by
-  rw [← IterM.reverse_toListRev]
-  simp
+  simp [← IterM.reverse_toListRev]
+
+theorem IterM.toListRev_ensureTermination [Monad m] [LawfulMonad m] [Iterator α m β] [Finite α m]
+    [IteratorCollect α m m] [LawfulIteratorCollect α m m]
+    {it : IterM (α := α) m β} :
+    it.ensureTermination.toListRev = List.reverse <$> it.toList := by
+  simp [← IterM.reverse_toListRev]
 
 theorem LawfulIteratorCollect.toArray_eq {α β : Type w} {m : Type w → Type w'}
     [Monad m] [Iterator α m β] [Finite α m] [IteratorCollect α m m]
     [hl : LawfulIteratorCollect α m m]
     {it : IterM (α := α) m β} :
     it.toArray = (letI : IteratorCollect α m m := .defaultImplementation; it.toArray) := by
-  simp only [IterM.toArray, toArrayMapped_eq]
+  simp [IterM.toArray, toArrayMapped_eq, IteratorCollect.defaultImplementation]
 
 theorem LawfulIteratorCollect.toList_eq {α β : Type w} {m : Type w → Type w'}
     [Monad m] [Iterator α m β] [Finite α m] [IteratorCollect α m m]

src/Init/Data/Iterators/Lemmas/Consumers/Monadic/Loop.lean

@@ -15,29 +15,75 @@ public section
 namespace Std.Iterators
 
 theorem IterM.DefaultConsumers.forIn'_eq_match_step {α β : Type w} {m : Type w → Type w'}
-    [Iterator α m β]
-    {n : Type x → Type x'} [Monad n]
+    [Iterator α m β] {n : Type x → Type x'} [Monad n] [LawfulMonad n]
     {lift : ∀ γ δ, (γ → n δ) → m γ → n δ} {γ : Type x}
-    {plausible_forInStep : β → γ → ForInStep γ → Prop}
-    {wf : IteratorLoop.WellFounded α m plausible_forInStep}
     {it : IterM (α := α) m β} {init : γ}
-    {P hP} {f : (b : β) → P b → (c : γ) → n (Subtype (plausible_forInStep b c))} :
-    IterM.DefaultConsumers.forIn' lift γ plausible_forInStep wf it init P hP f =
+    {P hP}
+    (PlausibleForInStep : β → γ → ForInStep γ → Prop)
+    {f : (b : β) → P b → (c : γ) → n (Subtype (PlausibleForInStep b c))}
+    (wf : IteratorLoop.WellFounded α m PlausibleForInStep) :
+    IterM.DefaultConsumers.forIn' lift γ PlausibleForInStep it init P hP f =
       (lift _ _ · it.step) (fun s =>
           match s.inflate with
           | .yield it' out h => do
             match ← f out (hP _ <| .direct ⟨_, h⟩) init with
             | ⟨.yield c, _⟩ =>
-              IterM.DefaultConsumers.forIn' lift _ plausible_forInStep wf it' c P
+              IterM.DefaultConsumers.forIn' lift _ PlausibleForInStep it' c P
                 (fun _ h' => hP _ <| .indirect ⟨_, rfl, h⟩ h') f
             | ⟨.done c, _⟩ => return c
           | .skip it' h =>
-            IterM.DefaultConsumers.forIn' lift _ plausible_forInStep wf it' init P
+            IterM.DefaultConsumers.forIn' lift _ PlausibleForInStep it' init P
               (fun _ h' => hP _ <| .indirect ⟨_, rfl, h⟩ h') f
           | .done _ => return init) := by
-  rw [forIn']
-  congr; ext step
-  cases step.inflate using PlausibleIterStep.casesOn <;> rfl
+  haveI : Nonempty γ := ⟨init⟩
+  rw [forIn', WellFounded.extrinsicFix₃_eq_apply]
+  · congr; ext step
+    cases step.inflate using PlausibleIterStep.casesOn
+    · simp only
+      apply bind_congr; intro step
+      split <;> simp [forIn']
+    · simp [forIn']
+    · simp
+  · apply InvImage.wf
+    exact wf
+
+theorem IterM.DefaultConsumers.forIn'_eq_wf {m : Type w → Type w'} {α : Type w} {β : Type w}
+    [Iterator α m β]
+    {n : Type x → Type x'} [Monad n] [LawfulMonad n]
+    {lift : ∀ γ δ, (γ → n δ) → m γ → n δ} {γ : Type x}
+    (Pl : β → γ → ForInStep γ → Prop)
+    (wf : IteratorLoop.WellFounded α m Pl)
+    {it : IterM (α := α) m β} {init : γ}
+    {P : β → Prop} {hP : ∀ b, it.IsPlausibleIndirectOutput b → P b}
+    (f : (b : β) → P b → (c : γ) → n (Subtype (Pl b c))) :
+    forIn' lift γ Pl it init P hP f =
+      forIn'.wf lift γ Pl wf it init P hP f := by
+  haveI : Nonempty γ := ⟨init⟩
+  rw [forIn', WellFounded.extrinsicFix₃_eq_fix]; rotate_left
+  · apply InvImage.wf
+    exact wf
+  · fun_induction forIn'.wf lift γ Pl wf it init P hP f
+    rename_i ihy ihs
+    rw [WellFounded.fix_eq]
+    congr 1; ext step
+    cases step.inflate using PlausibleIterStep.casesOn
+    · apply bind_congr; intro forInStep
+      match forInStep with
+      | ⟨.yield c, h⟩ => simp (discharger := assumption) [ihy]
+      | ⟨.done c, h⟩ => simp
+    · simp (discharger := assumption) [ihs]
+    · simp
+
+theorem IterM.DefaultConsumers.forIn'_eq_wf_of_finite {m : Type w → Type w'} {α : Type w}
+    {β : Type w} [Iterator α m β] [Finite α m]
+    {n : Type x → Type x'} [Monad n] [LawfulMonad n]
+    {lift : ∀ γ δ, (γ → n δ) → m γ → n δ} {γ : Type x}
+    {it : IterM (α := α) m β} {init : γ}
+    {P : β → Prop} {hP : ∀ b, it.IsPlausibleIndirectOutput b → P b}
+    (f : (b : β) → P b → (c : γ) → n (Subtype (fun _ => True))) :
+    forIn' lift γ (fun _ _ _ => True) it init P hP f =
+      forIn'.wf lift γ (fun _ _ _ => True) IteratorLoop.wellFounded_of_finite it init P hP f := by
+  apply forIn'_eq_wf
 
 theorem IterM.forIn'_eq {α β : Type w} {m : Type w → Type w'} [Iterator α m β] [Finite α m]
     {n : Type w → Type w''} [Monad m] [Monad n] [LawfulMonad n] [IteratorLoop α m n]
@@ -45,11 +91,12 @@ theorem IterM.forIn'_eq {α β : Type w} {m : Type w → Type w'} [Iterator α m
     [MonadLiftT m n] [LawfulMonadLiftT m n] {γ : Type w} {it : IterM (α := α) m β} {init : γ}
     {f : (b : β) → it.IsPlausibleIndirectOutput b → γ → n (ForInStep γ)} :
     letI : ForIn' n (IterM (α := α) m β) β _ := IterM.instForIn'
-    ForIn'.forIn' it init f = IterM.DefaultConsumers.forIn' (n := n)
-        (fun _ _ f x => monadLift x >>= f) γ (fun _ _ _ => True)
-        IteratorLoop.wellFounded_of_finite it init _ (fun _ => id) ((⟨·, .intro⟩) <$> f · · ·) := by
-  simp [instForIn', ForIn'.forIn', IteratorLoop.finiteForIn',
-    hl.lawful (fun _ _ f x => monadLift x >>= f), IteratorLoop.defaultImplementation]
+    ForIn'.forIn' (α := β) (m := n) it init f = IterM.DefaultConsumers.forIn' (n := n)
+        (fun _ _ f x => monadLift x >>= f) γ (fun _ _ _ => True) it init _ (fun _ => id) (return ⟨← f · · ·, trivial⟩) := by
+  simp only [instForIn', ForIn'.forIn', IteratorLoop.finiteForIn']
+  have : f = (Subtype.val <$> (⟨·, trivial⟩) <$> f · · ·) := by simp
+  rw [this, hl.lawful (fun _ _ f x => monadLift x >>= f) (wf := IteratorLoop.wellFounded_of_finite)]
+  simp [IteratorLoop.defaultImplementation]
 
 theorem IterM.forIn_eq {α β : Type w} {m : Type w → Type w'} [Iterator α m β] [Finite α m]
     {n : Type w → Type w''} [Monad m] [Monad n] [LawfulMonad n] [IteratorLoop α m n]
@@ -57,13 +104,13 @@ theorem IterM.forIn_eq {α β : Type w} {m : Type w → Type w'} [Iterator α m
     [MonadLiftT m n] [LawfulMonadLiftT m n] {γ : Type w} {it : IterM (α := α) m β} {init : γ}
     {f : β → γ → n (ForInStep γ)} :
     ForIn.forIn it init f = IterM.DefaultConsumers.forIn' (n := n)
-        (fun _ _ f x => monadLift x >>= f) γ (fun _ _ _ => True)
-        IteratorLoop.wellFounded_of_finite it init _ (fun _ => id) (fun out _ acc => (⟨·, .intro⟩) <$> f out acc) := by
+        (fun _ _ f x => monadLift x >>= f) γ (fun _ _ _ => True) it init _ (fun _ => id)
+        (fun out _ acc => return ⟨← f out acc, trivial⟩) := by
   simp only [ForIn.forIn, forIn'_eq]
 
 @[congr] theorem IterM.forIn'_congr {α β : Type w} {m : Type w → Type w'}
     {n : Type w → Type w''} [Monad n] [Monad m]
-    [Iterator α m β] [Finite α m] [IteratorLoop α m n] [MonadLiftT m n]
+    [Iterator α m β] [IteratorLoop α m n] [MonadLiftT m n]
     {ita itb : IterM (α := α) m β} (w : ita = itb)
     {b b' : γ} (hb : b = b')
     {f : (a' : β) → _ → γ → n (ForInStep γ)}
@@ -78,7 +125,7 @@ theorem IterM.forIn_eq {α β : Type w} {m : Type w → Type w'} [Iterator α m
 
 @[congr] theorem IterM.forIn_congr {α β : Type w} {m : Type w → Type w'}
     {n : Type w → Type w''} [Monad n] [Monad m]
-    [Iterator α m β] [Finite α m] [IteratorLoop α m n] [MonadLiftT m n]
+    [Iterator α m β] [IteratorLoop α m n] [MonadLiftT m n]
     {ita itb : IterM (α := α) m β} (w : ita = itb)
     {b b' : γ} (hb : b = b')
     {f : (a' : β) → γ → n (ForInStep γ)}
@@ -89,6 +136,36 @@ theorem IterM.forIn_eq {α β : Type w} {m : Type w → Type w'} [Iterator α m
   simp only [← funext_iff] at h
   rw [← h]
 
+theorem IterM.DefaultConsumers.forIn'_eq_forIn' {m : Type w → Type w'} {α : Type w} {β : Type w}
+    [Iterator α m β]
+    {n : Type x → Type x'} [Monad n] [LawfulMonad n]
+    {lift : ∀ γ δ, (γ → n δ) → m γ → n δ} {γ : Type x}
+    {it : IterM (α := α) m β} {init : γ}
+    {P : β → Prop} {hP : ∀ b, it.IsPlausibleIndirectOutput b → P b}
+    {Q : β → Prop} {hQ : ∀ b, it.IsPlausibleIndirectOutput b → Q b}
+    (Pl : β → γ → ForInStep γ → Prop)
+    {f : (b : β) → P b → (c : γ) → n (Subtype (Pl b c))}
+    {g : (b : β) → Q b → (c : γ) → n (Subtype (Pl b c))}
+    (wf : IteratorLoop.WellFounded α m Pl)
+    (hfg : ∀ b c, (hPb : P b) → (hQb : Q b) → f b hPb c = g b hQb c) :
+    IterM.DefaultConsumers.forIn' lift γ Pl it init P hP f =
+      IterM.DefaultConsumers.forIn' lift γ Pl it init Q hQ g := by
+  rw [forIn'_eq_match_step Pl wf, forIn'_eq_match_step Pl wf]
+  congr; ext step
+  split
+  · congr
+    · apply hfg
+    · ext forInStep
+      match forInStep with
+      | ⟨.yield _, h⟩ => apply IterM.DefaultConsumers.forIn'_eq_forIn' <;> assumption
+      | ⟨.done _, h⟩ => rfl
+  · apply IterM.DefaultConsumers.forIn'_eq_forIn' <;> assumption
+  · rfl
+termination_by IteratorLoop.WithWF.mk it init (hwf := wf)
+decreasing_by
+  · exact Or.inl ⟨_, ‹_›, ‹_›⟩
+  · exact Or.inr ⟨‹_›, rfl⟩
+
 theorem IterM.forIn'_eq_match_step {α β : Type w} {m : Type w → Type w'} [Iterator α m β]
     [Finite α m] {n : Type w → Type w''} [Monad m] [Monad n] [LawfulMonad n]
     [IteratorLoop α m n] [LawfulIteratorLoop α m n]
@@ -108,21 +185,21 @@ theorem IterM.forIn'_eq_match_step {α β : Type w} {m : Type w → Type w'} [It
           fun out h' acc => f out (.indirect ⟨_, rfl, h⟩ h') acc
       | .done _ => return init) := by
   rw [IterM.forIn'_eq, DefaultConsumers.forIn'_eq_match_step]
-  apply bind_congr
-  intro step
-  cases step.inflate using PlausibleIterStep.casesOn
-  · simp only [map_eq_pure_bind, bind_assoc]
-    apply bind_congr
-    intro forInStep
-    cases forInStep
+  · apply bind_congr; intro step
+    cases step.inflate using PlausibleIterStep.casesOn
+    · simp only [bind_assoc]
+      apply bind_congr
+      intro forInStep
+      cases forInStep
+      · simp
+      · simp only [forIn'_eq, pure_bind]
+        exact DefaultConsumers.forIn'_eq_forIn' (α := α) (m := m) (n := n) _
+          IteratorLoop.wellFounded_of_finite (by simp)
+    · simp only [forIn'_eq]
+      exact DefaultConsumers.forIn'_eq_forIn' (α := α) (m := m) (n := n) _
+          IteratorLoop.wellFounded_of_finite (by simp)
     · simp
-    · simp only [bind_pure_comp, pure_bind, forIn'_eq]
-      apply DefaultConsumers.forIn'_eq_forIn'
-      intros; congr
-  · simp only [forIn'_eq]
-    apply DefaultConsumers.forIn'_eq_forIn'
-    intros; congr
-  · simp
+  · exact IteratorLoop.wellFounded_of_finite
 
 theorem IterM.forIn_eq_match_step {α β : Type w} {m : Type w → Type w'} [Iterator α m β]
     [Finite α m] {n : Type w → Type w''} [Monad m] [Monad n] [LawfulMonad n]

src/Init/Data/Iterators/Producers/Monadic/List.lean

@@ -72,19 +72,9 @@ instance {α : Type w} [Monad m] {n : Type w → Type w''} [Monad n] :
     IteratorCollect (ListIterator α) m n :=
   .defaultImplementation
 
-@[always_inline, inline]
-instance {α : Type w} [Monad m] {n : Type w → Type w''} [Monad n] :
-    IteratorCollectPartial (ListIterator α) m n :=
-  .defaultImplementation
-
 @[always_inline, inline]
 instance {α : Type w} [Monad m] {n : Type x → Type x'} [Monad n] :
     IteratorLoop (ListIterator α) m n :=
   .defaultImplementation
 
-@[always_inline, inline]
-instance {α : Type w} [Monad m] {n : Type x → Type x'} [Monad n] :
-    IteratorLoopPartial (ListIterator α) m n :=
-  .defaultImplementation
-
 end Std.Iterators

src/Init/Data/List/Basic.lean

@@ -1607,8 +1607,8 @@ such element is found.
 `O(|l|)`.
 
 Examples:
-* `[7, 6, 5, 8, 1, 2, 6].find? (· < 5) = some 2`
-* `[7, 6, 5, 8, 1, 2, 6].find? (· < 1) = none`
+* `[7, 6, 5, 8, 1, 2, 6].findRev? (· < 5) = some 2`
+* `[7, 6, 5, 8, 1, 2, 6].findRev? (· < 1) = none`
 -/
 def findRev? (p : α → Bool) : List α → Option α
   | []    => none
@@ -2252,7 +2252,7 @@ def eraseReps {α} [BEq α] (as : List α) : List α := eraseRepsBy (· == ·) a
 /-! ### span -/
 
 /--
-Splits a list into the the longest initial segment for which `p` returns `true`, paired with the
+Splits a list into the longest initial segment for which `p` returns `true`, paired with the
 remainder of the list.
 
 `O(|l|)`.

src/Init/Data/List/Count.lean

@@ -29,7 +29,7 @@ section countP
 
 variable {p q : α → Bool}
 
-@[simp] theorem countP_nil : countP p [] = 0 := rfl
+@[simp, grind =] theorem countP_nil : countP p [] = 0 := rfl
 
 protected theorem countP_go_eq_add {l} : countP.go p l n = n + countP.go p l 0 := by
   induction l generalizing n with
@@ -47,6 +47,7 @@ protected theorem countP_go_eq_add {l} : countP.go p l n = n + countP.go p l 0 :
 @[simp] theorem countP_cons_of_neg {l} (pa : ¬p a) : countP p (a :: l) = countP p l := by
   simp [countP, countP.go, pa]
 
+@[grind =]
 theorem countP_cons {a : α} {l : List α} : countP p (a :: l) = countP p l + if p a then 1 else 0 := by
   by_cases h : p a <;> simp [h]
 
@@ -66,7 +67,6 @@ theorem length_eq_countP_add_countP (p : α → Bool) {l : List α} : length l =
       · rfl
       · simp [h]
 
-@[grind =]  -- This to quite aggressive, as it introduces `filter` based reasoning whenever we see `countP`.
 theorem countP_eq_length_filter {l : List α} : countP p l = (filter p l).length := by
   induction l with
   | nil => rfl
@@ -75,7 +75,8 @@ theorem countP_eq_length_filter {l : List α} : countP p l = (filter p l).length
     then rw [countP_cons_of_pos h, ih, filter_cons_of_pos h, length]
     else rw [countP_cons_of_neg h, ih, filter_cons_of_neg h]
 
-@[grind =]
+grind_pattern countP_eq_length_filter => l.countP p, l.filter p
+
 theorem countP_eq_length_filter' : countP p = length ∘ filter p := by
   funext l
   apply countP_eq_length_filter

src/Init/Data/List/Lemmas.lean

@@ -77,7 +77,7 @@ Further results, which first require developing further automation around `Nat`,
 * `Init.Data.List.Nat.TakeDrop`: `List.take` and `List.drop`
 
 Also
-* `Init.Data.List.Monadic` for addiation lemmas about `List.mapM` and `List.forM`.
+* `Init.Data.List.Monadic` for additional lemmas about `List.mapM` and `List.forM`.
 
 -/
 

src/Init/Data/List/Nat/Perm.lean

@@ -60,14 +60,14 @@ theorem set_set_perm {as : List α} {i j : Nat} (h₁ : i < as.length) (h₂ : j
 
 namespace Perm
 
-/-- Variant of `List.Perm.take` specifying the the permutation is constant after `i` elementwise. -/
+/-- Variant of `List.Perm.take` specifying that the permutation is constant after `i` elementwise. -/
 theorem take_of_getElem? {l₁ l₂ : List α} (h : l₁ ~ l₂) {i : Nat} (w : ∀ j, i ≤ j → l₁[j]? = l₂[j]?) :
     l₁.take i ~ l₂.take i := by
   refine h.take (Perm.of_eq ?_)
   ext1 j
   simpa using w (i + j) (by omega)
 
-/-- Variant of `List.Perm.drop` specifying the the permutation is constant before `i` elementwise. -/
+/-- Variant of `List.Perm.drop` specifying that the permutation is constant before `i` elementwise. -/
 theorem drop_of_getElem? {l₁ l₂ : List α} (h : l₁ ~ l₂) {i : Nat} (w : ∀ j, j < i → l₁[j]? = l₂[j]?) :
     l₁.drop i ~ l₂.drop i := by
   refine h.drop (Perm.of_eq ?_)

src/Init/Data/List/Range.lean

@@ -37,7 +37,7 @@ open Nat
   rw [← length_eq_zero_iff, length_range']
 
 theorem range'_ne_nil_iff (s : Nat) {n step : Nat} : range' s n step ≠ [] ↔ n ≠ 0 := by
-  cases n <;> simp
+  simp
 
 theorem range'_eq_cons_iff : range' s n step = a :: xs ↔ s = a ∧ 0 < n ∧ xs = range' (a + step) (n - 1) step := by
   induction n generalizing s with

src/Init/Data/List/Sublist.lean

@@ -249,7 +249,7 @@ theorem Sublist.eq_of_length : l₁ <+ l₂ → length l₁ = length l₂ → l
   | .cons a s, h => nomatch Nat.not_lt.2 s.length_le (h ▸ lt_succ_self _)
   | .cons₂ a s, h => by rw [s.eq_of_length (succ.inj h)]
 
--- Only activative `eq_of_length` if we're already thinking about lengths.
+-- Only activate `eq_of_length` if we're already thinking about lengths.
 grind_pattern Sublist.eq_of_length => l₁ <+ l₂, length l₁, length l₂
 
 theorem Sublist.eq_of_length_le (s : l₁ <+ l₂) (h : length l₂ ≤ length l₁) : l₁ = l₂ :=

src/Init/Data/Nat/Dvd.lean

@@ -27,9 +27,15 @@ protected theorem dvd_trans {a b c : Nat} (h₁ : a ∣ b) (h₂ : b ∣ c) : a
 protected theorem dvd_mul_left_of_dvd {a b : Nat} (h : a ∣ b) (c : Nat) : a ∣ c * b :=
   Nat.dvd_trans h (Nat.dvd_mul_left _ _)
 
+grind_pattern Nat.dvd_mul_left_of_dvd => a ∣ b, c * b where
+  guard a ∣ b
+
 protected theorem dvd_mul_right_of_dvd {a b : Nat} (h : a ∣ b) (c : Nat) : a ∣ b * c :=
   Nat.dvd_trans h (Nat.dvd_mul_right _ _)
 
+grind_pattern Nat.dvd_mul_right_of_dvd => a ∣ b, b * c where
+  guard a ∣ b
+
 protected theorem eq_zero_of_zero_dvd {a : Nat} (h : 0 ∣ a) : a = 0 :=
   let ⟨c, H'⟩ := h; H'.trans c.zero_mul
 

src/Init/Data/Nat/Lemmas.lean

@@ -1086,6 +1086,18 @@ protected theorem pow_add (a m n : Nat) : a ^ (m + n) = a ^ m * a ^ n := by
   | zero => rw [Nat.add_zero, Nat.pow_zero, Nat.mul_one]
   | succ _ ih => rw [Nat.add_succ, Nat.pow_succ, Nat.pow_succ, ih, Nat.mul_assoc]
 
+theorem div_pow_of_pos (a n : Nat) : n > 0 → a ∣ a ^ n := by
+  cases n <;> simp [Nat.pow_add]
+  exact Nat.dvd_mul_left a (a ^ _)
+
+grind_pattern div_pow_of_pos => a ^ n where
+  is_value a
+  guard n > 0
+
+grind_pattern Nat.pow_pos => a ^ n where
+  not_value n
+  guard a > 0
+
 protected theorem pow_add' (a m n : Nat) : a ^ (m + n) = a ^ n * a ^ m := by
   rw [← Nat.pow_add, Nat.add_comm]
 

src/Init/Data/Range/Polymorphic/Lemmas.lean

@@ -467,6 +467,23 @@ public theorem Rxo.Iterator.toArray_eq_match [LT α] [DecidableLT α]
   · rfl
   · split <;> simp
 
+public theorem Rxc.Iterator.toList_eq_toList_rxoIterator [LE α] [DecidableLE α] [LT α] [DecidableLT α]
+    [UpwardEnumerable α] [Rxc.IsAlwaysFinite α] [Rxo.IsAlwaysFinite α] [LawfulUpwardEnumerable α]
+    [LawfulUpwardEnumerableLE α] [LawfulUpwardEnumerableLT α]
+    [InfinitelyUpwardEnumerable α] [LinearlyUpwardEnumerable α] {it : Iter (α := Rxc.Iterator α) α}:
+    it.toList = (⟨⟨it.internalState.next, succ it.internalState.upperBound⟩⟩ : Iter (α := Rxo.Iterator α) α).toList := by
+  induction it using Iter.inductSteps with | step it ihy ihs
+  rw [Rxc.Iterator.toList_eq_match, Rxo.Iterator.toList_eq_match]
+  split
+  · simp [*]
+  · simp only [UpwardEnumerable.le_iff, UpwardEnumerable.lt_iff, *]
+    split <;> rename_i h
+    · rw [ihy]; rotate_left
+      · simp [Iter.IsPlausibleStep, IterM.IsPlausibleStep, Iterator.IsPlausibleStep,
+          Iterator.Monadic.step, Iter.toIterM, *]; rfl
+      · simpa [UpwardEnumerable.lt_iff, UpwardEnumerable.le_iff, UpwardEnumerable.lt_succ_iff] using h
+    · simpa [UpwardEnumerable.lt_iff, UpwardEnumerable.le_iff, UpwardEnumerable.lt_succ_iff] using h
+
 public theorem Rxi.Iterator.toList_eq_match
     [UpwardEnumerable α] [Rxi.IsAlwaysFinite α] [LawfulUpwardEnumerable α]
     {it : Iter (α := Rxi.Iterator α) α} :
@@ -561,22 +578,6 @@ namespace Rcc
 
 variable {r : Rcc α}
 
-public theorem toList_eq_if_roo [UpwardEnumerable α] [LE α] [DecidableLE α]
-    [LawfulUpwardEnumerable α] [Rxc.IsAlwaysFinite α] [LawfulUpwardEnumerableLE α] :
-    r.toList = if r.lower ≤ r.upper then r.lower :: (r.lower<...=r.upper).toList else [] := by
-  rw [Internal.toList_eq_toList_iter, Rxc.Iterator.toList_eq_match]; rfl
-
-@[deprecated toList_eq_if_roo (since := "2025-10-29")]
-def toList_eq_if_Roo := @toList_eq_if_roo
-
-public theorem toArray_eq_if_roo [UpwardEnumerable α] [LE α] [DecidableLE α]
-    [LawfulUpwardEnumerable α] [Rxc.IsAlwaysFinite α] [LawfulUpwardEnumerableLE α] :
-    r.toArray = if r.lower ≤ r.upper then #[r.lower] ++ (r.lower<...=r.upper).toArray else #[] := by
-  rw [Internal.toArray_eq_toArray_iter, Rxc.Iterator.toArray_eq_match]; rfl
-
-@[deprecated toArray_eq_if_roo (since := "2025-10-29")]
-def toArray_eq_if_Roo := @toArray_eq_if_roo
-
 public theorem toList_eq_if_roc [LE α] [DecidableLE α] [UpwardEnumerable α]
     [LawfulUpwardEnumerable α] [LawfulUpwardEnumerableLE α] [Rxc.IsAlwaysFinite α] :
     r.toList = if r.lower ≤ r.upper then
@@ -585,6 +586,16 @@ public theorem toList_eq_if_roc [LE α] [DecidableLE α] [UpwardEnumerable α]
         [] := by
   rw [Internal.toList_eq_toList_iter, Rxc.Iterator.toList_eq_match]; rfl
 
+@[simp]
+public theorem toList_eq_toList_rco [LE α] [DecidableLE α] [LT α] [DecidableLT α]
+    [UpwardEnumerable α] [LawfulUpwardEnumerable α]
+    [LawfulUpwardEnumerableLE α] [LawfulUpwardEnumerableLT α]
+    [Rxc.IsAlwaysFinite α] [Rxo.IsAlwaysFinite α]
+    [InfinitelyUpwardEnumerable α] [LinearlyUpwardEnumerable α] :
+    r.toList = (r.lower...(succ r.upper)).toList := by
+  simp [Internal.toList_eq_toList_iter, Rco.Internal.toList_eq_toList_iter,
+    Internal.iter, Rco.Internal.iter, Rxc.Iterator.toList_eq_toList_rxoIterator]
+
 @[deprecated toList_eq_if_roc (since := "2025-10-29")]
 def toList_eq_match := @toList_eq_if_roc
 
@@ -816,6 +827,23 @@ public theorem toArray_eq_if_roo [UpwardEnumerable α] [LT α] [DecidableLT α]
         #[] := by
   rw [Internal.toArray_eq_toArray_iter, Rxo.Iterator.toArray_eq_match]; rfl
 
+public theorem toList_eq_if_rco [UpwardEnumerable α] [LT α] [DecidableLT α]
+    [LawfulUpwardEnumerable α] [Rxo.IsAlwaysFinite α] [LawfulUpwardEnumerableLT α] :
+    r.toList = if r.lower < r.upper then
+        match UpwardEnumerable.succ? r.lower with
+        | none => [r.lower]
+        | some next => r.lower :: (next...r.upper).toList
+      else
+        [] := by
+  rw [Internal.toList_eq_toList_iter, Rxo.Iterator.toList_eq_match]
+  simp only [Internal.iter]
+  split
+  · split
+    · simp [Rxo.Iterator.toList_eq_match, *]
+    · simp only [*]
+      rfl
+  · rfl
+
 public theorem toArray_eq_if_rco [UpwardEnumerable α] [LT α] [DecidableLT α]
     [LawfulUpwardEnumerable α] [Rxo.IsAlwaysFinite α] [LawfulUpwardEnumerableLT α] :
     r.toArray = if r.lower < r.upper then
@@ -1272,6 +1300,16 @@ public theorem toArray_eq_match_rcc [LE α] [DecidableLE α] [UpwardEnumerable
   simp only [← Internal.toList_eq_toList_iter, toList_eq_match_rcc]
   split <;> simp
 
+@[simp]
+public theorem toList_eq_toList_roo [LE α] [DecidableLE α] [LT α] [DecidableLT α]
+    [UpwardEnumerable α] [LawfulUpwardEnumerable α]
+    [LawfulUpwardEnumerableLE α] [LawfulUpwardEnumerableLT α]
+    [Rxc.IsAlwaysFinite α] [Rxo.IsAlwaysFinite α]
+    [InfinitelyUpwardEnumerable α] [LinearlyUpwardEnumerable α] :
+    r.toList = (r.lower<...(succ r.upper)).toList := by
+  simp [Internal.toList_eq_toList_iter, Roo.Internal.toList_eq_toList_iter,
+    Internal.iter, Roo.Internal.iter, Rxc.Iterator.toList_eq_toList_rxoIterator]
+
 @[simp]
 public theorem toArray_toList [LE α] [DecidableLE α] [UpwardEnumerable α] [LawfulUpwardEnumerable α]
     [Rxc.IsAlwaysFinite α] :
@@ -2856,7 +2894,7 @@ public theorem length_toList [LE α] [DecidableLE α] [UpwardEnumerable α]
   · simpa [toList_eq_nil_iff, size_eq_if_roc] using h
   · rename_i n ih
     rw [size_eq_if_rcc] at h
-    simp only [toList_eq_if_roo, ← h]
+    simp only [toList_eq_if_roc, ← h]
     simp only [Roc.toList_eq_match_rcc]
     split
     · split

src/Init/Data/Range/Polymorphic/RangeIterator.lean

@@ -8,8 +8,10 @@ module
 prelude
 public import Init.Data.Iterators.Internal.Termination
 public import Init.Data.Iterators.Consumers.Access
+import Init.Data.Iterators.Lemmas.Consumers.Monadic.Loop
 public import Init.Data.Range.Polymorphic.PRange
 public import Init.Data.List.Sublist
+public import Init.WFExtrinsicFix
 
 set_option doc.verso true
 
@@ -121,10 +123,6 @@ instance Iterator.instIteratorCollect [UpwardEnumerable α] [LE α] [DecidableLE
     {n : Type u → Type w} [Monad n] : IteratorCollect (Rxc.Iterator α) Id n :=
   .defaultImplementation
 
-instance Iterator.instIteratorCollectPartial [UpwardEnumerable α] [LE α] [DecidableLE α]
-    {n : Type u → Type w} [Monad n] : IteratorCollectPartial (Rxc.Iterator α) Id n :=
-  .defaultImplementation
-
 theorem Iterator.Monadic.isPlausibleOutput_next {a}
     [UpwardEnumerable α] [LE α] [DecidableLE α]
     {it : IterM (α := Rxc.Iterator α) Id α} (h : it.internalState.next = some a)
@@ -448,161 +446,168 @@ instance Iterator.instIteratorLoop [UpwardEnumerable α] [LE α] [DecidableLE α
     [LawfulUpwardEnumerable α] [LawfulUpwardEnumerableLE α]
     {n : Type u → Type w} [Monad n] :
     IteratorLoop (Rxc.Iterator α) Id n where
-  forIn _ γ Pl wf it init f :=
+  forIn _ γ Pl it init f :=
     match it with
     | ⟨⟨some next, upperBound⟩⟩ =>
-      if hu : next ≤ upperBound then
-        loop γ Pl wf upperBound next init (fun a ha₁ ha₂ c => f a ?hf c) next ?hle hu
-      else
-        return init
+      loop γ Pl (next ≤ ·) (fun a b hab hna => ?hle) upperBound init next ?hle'' (fun a ha₁ ha₂ c => f a ?hf c)
     | ⟨⟨none, _⟩⟩ => return init
   where
-    @[specialize]
-    loop γ Pl wf (upperBound : α) least acc
-        (f : (out : α) → UpwardEnumerable.LE least out → out ≤ upperBound → (c : γ) → n (Subtype (fun s : ForInStep γ => Pl out c s)))
-        (next : α) (hl : UpwardEnumerable.LE least next) (hu : next ≤ upperBound) : n γ := do
-      match ← f next hl hu acc with
-      | ⟨.yield acc', _⟩ =>
-        match hs : UpwardEnumerable.succ? next with
-        | some next' =>
-          if hu : next' ≤ upperBound then
-            loop γ Pl wf upperBound least acc' f next' ?hle' hu
+    @[always_inline, inline]
+    loop γ (Pl : α → γ → ForInStep γ → Prop) (LargeEnough : α → Prop) (hl : ∀ a b : α, a ≤ b → LargeEnough a → LargeEnough b)
+        (upperBound : α) (acc : γ) (next : α) (h : LargeEnough next)
+        (f : (out : α) → LargeEnough out → out ≤ upperBound → (c : γ) → n (Subtype (Pl out c))) : n γ :=
+      haveI : Nonempty γ := ⟨acc⟩
+      WellFounded.extrinsicFix₃ (C₃ := fun _ _ _ => n γ) (InvImage (IteratorLoop.rel _ Id Pl) (fun x => (⟨Rxc.Iterator.mk (some x.1) upperBound⟩, x.2.1)))
+        (fun next acc (h : LargeEnough next) G => do
+          if hu : next ≤ upperBound then
+            match ← f next h hu acc with
+            | ⟨.yield acc', h'⟩ =>
+              match hs : UpwardEnumerable.succ? next with
+              | some next' => G next' acc' (hl _ _ ?hle' h) ?decreasing
+              | none => return acc'
+            | ⟨.done acc', _⟩ => return acc'
           else
-            return acc'
-        | none => return acc'
-      | ⟨.done acc', _⟩ => return acc'
-    termination_by IteratorLoop.WithWF.mk ⟨⟨some next, upperBound⟩⟩ acc (hwf := wf)
-    decreasing_by
-      simp [IteratorLoop.rel, Monadic.isPlausibleStep_iff,
-        Monadic.step, *]
+            return acc) next acc h
   finally
     case hf =>
       rw [Monadic.isPlausibleIndirectOutput_iff]
+      simp only [UpwardEnumerable.le_iff] at ha₁
       obtain ⟨n, hn⟩ := ha₁
       exact ⟨n, hn, ha₂⟩
     case hle =>
-      exact UpwardEnumerable.le_refl _
+      simp only [UpwardEnumerable.le_iff] at hna hab ⊢
+      exact UpwardEnumerable.le_trans hna hab
     case hle' =>
-      refine UpwardEnumerable.le_trans hl ⟨1, ?_⟩
-      simp [succMany?_one, hs]
+      simp only [UpwardEnumerable.le_iff]
+      refine ⟨1, ?_⟩
+      simpa [succMany?_one] using hs
+    case hle'' =>
+      exact UpwardEnumerable.le_iff.mpr (UpwardEnumerable.le_refl _)
+    case decreasing =>
+      simp_wf
+      simp [IteratorLoop.rel, Monadic.isPlausibleStep_iff, Monadic.step, *]
 
-/--
-An efficient {name}`IteratorLoop` instance:
-As long as the compiler cannot optimize away the {name}`Option` in the internal state, we use a special
-loop implementation.
--/
-partial instance Iterator.instIteratorLoopPartial [UpwardEnumerable α] [LE α] [DecidableLE α]
+private noncomputable def Iterator.instIteratorLoop.loop.wf [UpwardEnumerable α] [LE α] [DecidableLE α]
     [LawfulUpwardEnumerable α] [LawfulUpwardEnumerableLE α]
-    {n : Type u → Type w} [Monad n] : IteratorLoopPartial (Rxc.Iterator α) Id n where
-  forInPartial _ γ it init f :=
-    match it with
-    | ⟨⟨some next, upperBound⟩⟩ =>
+    {n : Type u → Type w} [Monad n] (γ : Type u)
+    (Pl : α → γ → ForInStep γ → Prop)
+    (wf : IteratorLoop.WellFounded (Rxc.Iterator α) Id Pl)
+    (LargeEnough : α → Prop) (hl : ∀ a b : α, a ≤ b → LargeEnough a → LargeEnough b)
+    (upperBound : α) (acc : γ) (next : α) (h : LargeEnough next)
+    (f : (out : α) → LargeEnough out → out ≤ upperBound → (c : γ) → n (Subtype (fun s : ForInStep γ => Pl out c s))) :
+    n γ := do
+  if hu : next ≤ upperBound then
+    match ← f next h hu acc with
+    | ⟨.yield acc', _⟩ =>
+      match hs : UpwardEnumerable.succ? next with
+      | some next' =>
+        loop.wf γ Pl wf LargeEnough hl upperBound acc' next' (hl _ _ ?hle h) f
+      | none => return acc'
+    | ⟨.done acc', _⟩ => return acc'
+  else
+    return acc
+termination_by IteratorLoop.WithWF.mk ⟨⟨some next, upperBound⟩⟩ acc (hwf := wf)
+decreasing_by
+  simp [IteratorLoop.rel, Monadic.isPlausibleStep_iff, Monadic.step, *]
+where finally
+  case hle =>
+    simp only [UpwardEnumerable.le_iff]
+    refine ⟨1, ?_⟩
+    simpa [succMany?_one] using hs
+
+private theorem Iterator.instIteratorLoop.loop_eq_wf [UpwardEnumerable α] [LE α] [DecidableLE α]
+    [LawfulUpwardEnumerable α] [LawfulUpwardEnumerableLE α] [Monad n] [LawfulMonad n]
+    {γ LargeEnough hl upperBound} {next hn} {acc} (Pl wf f) :
+    loop γ Pl LargeEnough hl upperBound acc next hn f =
+      loop.wf (α := α) (n := n) γ Pl wf LargeEnough hl upperBound acc next hn f := by
+  haveI : Nonempty γ := ⟨acc⟩
+  rw [loop, WellFounded.extrinsicFix₃_eq_fix]; rotate_left
+  · exact InvImage.wf _ wf
+  · fun_induction loop.wf γ Pl wf LargeEnough hl upperBound acc  next hn f
+    · rw [WellFounded.fix_eq]
+      simp only [↓reduceDIte, *]
+      apply bind_congr; intro forInStep
+      split
+      · simp only
+        split
+        · simp_all
+        · simp
+      · simp
+    · rw [WellFounded.fix_eq]
+      simp_all
+
+private theorem Iterator.instIteratorLoop.loopWf_eq [UpwardEnumerable α] [LE α] [DecidableLE α]
+    [LawfulUpwardEnumerable α] [LawfulUpwardEnumerableLE α]
+    {n : Type u → Type w} [Monad n] [LawfulMonad n] (γ : Type u)
+    {lift} [instLawfulMonadLiftFunction : Std.Internal.LawfulMonadLiftBindFunction (m := Id) (n := n) lift]
+    (Pl : α → γ → ForInStep γ → Prop)
+    (wf : IteratorLoop.WellFounded (Rxc.Iterator α) Id Pl)
+    (LargeEnough : α → Prop) (hl : ∀ a b : α, a ≤ b → LargeEnough a → LargeEnough b)
+    (upperBound : α) (acc : γ) (next : α) (h : LargeEnough next)
+    (f : (out : α) → LargeEnough out → out ≤ upperBound → (c : γ) → n (Subtype (fun s : ForInStep γ => Pl out c s))) :
+    loop.wf γ Pl wf LargeEnough hl upperBound acc next h f = (do
       if hu : next ≤ upperBound then
-        loop γ upperBound next init (fun a ha₁ ha₂ c => f a ?hf c) next ?hle hu
-      else
-        return init
-    | ⟨⟨none, _⟩⟩ => return init
-  where
-    @[specialize]
-    loop γ (upperBound : α) least acc
-        (f : (out : α) → UpwardEnumerable.LE least out → out ≤ upperBound → (c : γ) → n (ForInStep γ))
-        (next : α) (hl : UpwardEnumerable.LE least next) (hu : next ≤ upperBound) : n γ := do
-      match ← f next hl hu acc with
-      | .yield acc' =>
-        match hs : succ? next with
-        | some next' =>
-          if hu : next' ≤ upperBound then
-            loop γ upperBound least acc' f next' ?hle' hu
-          else
-            return acc'
-        | none => return acc'
-      | .done acc' => return acc'
-  finally
-    case hf =>
-      rw [Monadic.isPlausibleIndirectOutput_iff]
-      obtain ⟨n, hn⟩ := ha₁
-      exact ⟨n, hn, ha₂⟩
-    case hle =>
-      exact UpwardEnumerable.le_refl _
-    case hle' =>
-      refine UpwardEnumerable.le_trans hl ⟨1, ?_⟩
-      simp [succMany?_one, hs]
-
-theorem Iterator.instIteratorLoop.loop_eq [UpwardEnumerable α] [LE α] [DecidableLE α]
-    [LawfulUpwardEnumerable α] [LawfulUpwardEnumerableLE α]
-    {n : Type u → Type w} [Monad n] [LawfulMonad n] {γ : Type u}
-    {lift} [Internal.LawfulMonadLiftBindFunction lift]
-    {PlausibleForInStep} {upperBound} {next} {hl} {hu} {f} {acc} {wf} :
-    loop (α := α) (n := n) γ PlausibleForInStep wf upperBound least acc f next hl hu =
-      (do
-        match ← f next hl hu acc with
-        | ⟨.yield c, _⟩ =>
+        match ← f next h hu acc with
+        | ⟨.yield acc', _⟩ =>
           letI it' : IterM (α := Rxc.Iterator α) Id α := ⟨⟨succ? next, upperBound⟩⟩
-          IterM.DefaultConsumers.forIn' (m := Id) lift γ
-            PlausibleForInStep wf it' c it'.IsPlausibleIndirectOutput (fun _ => id)
-            (fun b h c => f b
-                (by
-                  refine UpwardEnumerable.le_trans hl ?_
-                  simp only [Monadic.isPlausibleIndirectOutput_iff, it',
-                    ← succMany?_add_one_eq_succ?_bind_succMany?] at h
-                  exact ⟨h.choose + 1, h.choose_spec.1⟩)
-                (by
-                  simp only [Monadic.isPlausibleIndirectOutput_iff, it'] at h
-                  exact h.choose_spec.2) c)
-        | ⟨.done c, _⟩ => return c) := by
-  rw [loop]
-  apply bind_congr
-  intro step
+          IterM.DefaultConsumers.forIn' (m := Id) (n := n) lift γ Pl it' acc'
+            it'.IsPlausibleIndirectOutput (fun _ => id)
+            fun next' h acc' => f next'
+              (by
+                refine hl next next' ?_ ‹_›
+                simp only [it', Monadic.isPlausibleIndirectOutput_iff,
+                  ← succMany?_add_one_eq_succ?_bind_succMany?] at h
+                exact UpwardEnumerable.le_iff.mpr ⟨h.choose + 1, h.choose_spec.1⟩)
+              (by
+                simp only [it', Monadic.isPlausibleIndirectOutput_iff] at h
+                exact h.choose_spec.2)
+              acc'
+        | ⟨.done acc', _⟩ => return acc'
+      else return acc) := by
+  haveI : Nonempty γ := ⟨acc⟩
+  rw [loop.wf]
+  congr 1; ext hu
+  apply bind_congr; intro forInStep
   split
   · split
-    · split
-      · simp only [*]
-        rw [IterM.DefaultConsumers.forIn']
-        simp only [Monadic.step_eq_step, Monadic.step, ↓reduceIte, *,
-          Internal.LawfulMonadLiftBindFunction.liftBind_pure]
-        rw [loop_eq (lift := lift), Shrink.inflate_deflate]
-        apply bind_congr
-        intro step
+    · rw [loopWf_eq (lift := lift) _ Pl wf]
+      rw [IterM.DefaultConsumers.forIn'_eq_match_step (lift := lift) Pl wf]; rotate_left
+      · simp only [Monadic.step_eq_step, Monadic.step,
+          Shrink.inflate_deflate, instLawfulMonadLiftFunction.liftBind_pure, *]
         split
-        · apply IterM.DefaultConsumers.forIn'_eq_forIn'
-          intros; rfl
+        · apply bind_congr; intro forInStep
+          split
+          · apply IterM.DefaultConsumers.forIn'_eq_forIn' Pl wf <;> (intros; rfl)
+          · simp
         · simp
-      · simp only [*]
-        rw [IterM.DefaultConsumers.forIn']
-        simp [Monadic.step_eq_step, Monadic.step, *,
-          Internal.LawfulMonadLiftBindFunction.liftBind_pure]
-    · simp only [*]
-      rw [IterM.DefaultConsumers.forIn']
-      simp [Monadic.step_eq_step, Monadic.step, Internal.LawfulMonadLiftBindFunction.liftBind_pure]
+    · rw [IterM.DefaultConsumers.forIn'_eq_match_step Pl wf]
+      simp [Monadic.step_eq_step, Monadic.step, instLawfulMonadLiftFunction.liftBind_pure, *]
   · simp
 termination_by IteratorLoop.WithWF.mk ⟨⟨some next, upperBound⟩⟩ acc (hwf := wf)
 decreasing_by
-      simp [IteratorLoop.rel, Monadic.isPlausibleStep_iff, Monadic.step, *]
+  simp [IteratorLoop.rel, Monadic.isPlausibleStep_iff, Monadic.step, *]
 
 instance Iterator.instLawfulIteratorLoop [UpwardEnumerable α] [LE α] [DecidableLE α]
     [LawfulUpwardEnumerable α] [LawfulUpwardEnumerableLE α]
     {n : Type u → Type w} [Monad n] [LawfulMonad n] :
     LawfulIteratorLoop (Rxc.Iterator α) Id n where
   lawful := by
-    intro lift instLawfulMonadLiftFunction
-    ext γ PlausibleForInStep hwf it init f
-    simp only [IteratorLoop.forIn, IteratorLoop.defaultImplementation]
-    rw [IterM.DefaultConsumers.forIn']
-    simp only [Monadic.step_eq_step, Monadic.step]
-    simp only [Internal.LawfulMonadLiftBindFunction.liftBind_pure]
+    intro lift instLawfulMonadLiftFunction γ it init Pl wf f
+    simp only [IteratorLoop.defaultImplementation, IteratorLoop.forIn,
+      IterM.DefaultConsumers.forIn'_eq_wf Pl wf]
+    rw [IterM.DefaultConsumers.forIn'.wf]
+    split; rotate_left
+    · simp [Monadic.step_eq_step, Monadic.step, Internal.LawfulMonadLiftBindFunction.liftBind_pure]
+    rename_i next _
+    rw [instIteratorLoop.loop_eq_wf Pl wf, instIteratorLoop.loopWf_eq (lift := lift)]
+    simp only [Monadic.step_eq_step, Monadic.step, instLawfulMonadLiftFunction.liftBind_pure,
+      Shrink.inflate_deflate]
     split
-    · rename_i it f next upperBound f'
-      simp
+    · apply bind_congr; intro forInStep
       split
       · simp only
-        rw [instIteratorLoop.loop_eq (lift := lift)]
-        apply bind_congr
-        intro step
-        split
-        · apply IterM.DefaultConsumers.forIn'_eq_forIn'
-          intro b c hPb hQb
-          congr
-        · simp
+        rw [← IterM.DefaultConsumers.forIn'_eq_wf Pl wf _]
+        apply IterM.DefaultConsumers.forIn'_eq_forIn' Pl wf <;> all_goals (intros; rfl)
       · simp
     · simp
 
@@ -698,10 +703,6 @@ instance Iterator.instIteratorCollect [UpwardEnumerable α] [LT α] [DecidableLT
     {n : Type u → Type w} [Monad n] : IteratorCollect (Rxo.Iterator α) Id n :=
   .defaultImplementation
 
-instance Iterator.instIteratorCollectPartial [UpwardEnumerable α] [LT α] [DecidableLT α]
-    {n : Type u → Type w} [Monad n] : IteratorCollectPartial (Rxo.Iterator α) Id n :=
-  .defaultImplementation
-
 theorem Iterator.Monadic.isPlausibleOutput_next {a}
     [UpwardEnumerable α] [LT α] [DecidableLT α]
     {it : IterM (α := Rxo.Iterator α) Id α} (h : it.internalState.next = some a)
@@ -1025,161 +1026,164 @@ instance Iterator.instIteratorLoop [UpwardEnumerable α] [LT α] [DecidableLT α
     [LawfulUpwardEnumerable α] [LawfulUpwardEnumerableLT α]
     {n : Type u → Type w} [Monad n] :
     IteratorLoop (Rxo.Iterator α) Id n where
-  forIn _ γ Pl wf it init f :=
+  forIn _ γ Pl it init f :=
     match it with
     | ⟨⟨some next, upperBound⟩⟩ =>
-      if hu : next < upperBound then
-        loop γ Pl wf upperBound next init (fun a ha₁ ha₂ c => f a ?hf c) next ?hle hu
-      else
-        return init
+      loop γ Pl (UpwardEnumerable.LE next ·) (fun a b hab hna => ?hle) upperBound init next ?hle'' (fun a ha₁ ha₂ c => f a ?hf c)
     | ⟨⟨none, _⟩⟩ => return init
   where
-    @[specialize]
-    loop γ Pl wf (upperBound : α) least acc
-        (f : (out : α) → UpwardEnumerable.LE least out → out < upperBound → (c : γ) → n (Subtype (fun s : ForInStep γ => Pl out c s)))
-        (next : α) (hl : UpwardEnumerable.LE least next) (hu : next < upperBound) : n γ := do
-      match ← f next hl hu acc with
-      | ⟨.yield acc', _⟩ =>
-        match hs : UpwardEnumerable.succ? next with
-        | some next' =>
-          if hu : next' < upperBound then
-            loop γ Pl wf upperBound least acc' f next' ?hle' hu
+    @[always_inline, inline]
+    loop γ (Pl : α → γ → ForInStep γ → Prop) (LargeEnough : α → Prop)
+        (hl : ∀ a b : α, UpwardEnumerable.LE a b → LargeEnough a → LargeEnough b)
+        (upperBound : α) (acc : γ) (next : α) (h : LargeEnough next)
+        (f : (out : α) → LargeEnough out → out < upperBound → (c : γ) → n (Subtype (Pl out c))) : n γ :=
+      haveI : Nonempty γ := ⟨acc⟩
+      WellFounded.extrinsicFix₃ (C₃ := fun _ _ _ => n γ) (InvImage (IteratorLoop.rel _ Id Pl) (fun x => (⟨Rxo.Iterator.mk (some x.1) upperBound⟩, x.2.1)))
+        (fun next acc (h : LargeEnough next) G => do
+          if hu : next < upperBound then
+            match ← f next h hu acc with
+            | ⟨.yield acc', h'⟩ =>
+              match hs : UpwardEnumerable.succ? next with
+              | some next' => G next' acc' (hl _ _ ?hle' h) ?decreasing
+              | none => return acc'
+            | ⟨.done acc', _⟩ => return acc'
           else
-            return acc'
-        | none => return acc'
-      | ⟨.done acc', _⟩ => return acc'
-    termination_by IteratorLoop.WithWF.mk ⟨⟨some next, upperBound⟩⟩ acc (hwf := wf)
-    decreasing_by
-      simp [IteratorLoop.rel, Monadic.isPlausibleStep_iff,
-        Monadic.step, *]
+            return acc) next acc h
   finally
     case hf =>
       rw [Monadic.isPlausibleIndirectOutput_iff]
       obtain ⟨n, hn⟩ := ha₁
       exact ⟨n, hn, ha₂⟩
     case hle =>
-      exact UpwardEnumerable.le_refl _
+      exact UpwardEnumerable.le_trans hna hab
     case hle' =>
-      refine UpwardEnumerable.le_trans hl ⟨1, ?_⟩
-      simp [succMany?_one, hs]
+      refine ⟨1, ?_⟩
+      simpa [succMany?_one] using hs
+    case hle'' =>
+      exact UpwardEnumerable.le_refl _
+    case decreasing =>
+      simp_wf; simp [IteratorLoop.rel, Monadic.isPlausibleStep_iff, Monadic.step, *]
 
-/--
-An efficient {name}`IteratorLoopPartial` instance:
-As long as the compiler cannot optimize away the {name}`Option` in the internal state, we use a special
-loop implementation.
--/
-partial instance Iterator.instIteratorLoopPartial [UpwardEnumerable α] [LT α] [DecidableLT α]
+private noncomputable def Iterator.instIteratorLoop.loop.wf [UpwardEnumerable α] [LT α] [DecidableLT α]
     [LawfulUpwardEnumerable α] [LawfulUpwardEnumerableLT α]
-    {n : Type u → Type w} [Monad n] : IteratorLoopPartial (Rxo.Iterator α) Id n where
-  forInPartial _ γ it init f :=
-    match it with
-    | ⟨⟨some next, upperBound⟩⟩ =>
+    {n : Type u → Type w} [Monad n] (γ : Type u)
+    (Pl : α → γ → ForInStep γ → Prop)
+    (wf : IteratorLoop.WellFounded (Rxo.Iterator α) Id Pl)
+    (LargeEnough : α → Prop) (hl : ∀ a b : α, UpwardEnumerable.LE a b → LargeEnough a → LargeEnough b)
+    (upperBound : α) (acc : γ) (next : α) (h : LargeEnough next)
+    (f : (out : α) → LargeEnough out → out < upperBound → (c : γ) → n (Subtype (fun s : ForInStep γ => Pl out c s))) :
+    n γ := do
+  if hu : next < upperBound then
+    match ← f next h hu acc with
+    | ⟨.yield acc', _⟩ =>
+      match hs : UpwardEnumerable.succ? next with
+      | some next' =>
+        loop.wf γ Pl wf LargeEnough hl upperBound acc' next' (hl _ _ ?hle h) f
+      | none => return acc'
+    | ⟨.done acc', _⟩ => return acc'
+  else
+    return acc
+termination_by IteratorLoop.WithWF.mk ⟨⟨some next, upperBound⟩⟩ acc (hwf := wf)
+decreasing_by
+  simp [IteratorLoop.rel, Monadic.isPlausibleStep_iff, Monadic.step, *]
+where finally
+  case hle =>
+    refine ⟨1, ?_⟩
+    simpa [succMany?_one] using hs
+
+private theorem Iterator.instIteratorLoop.loop_eq_wf [UpwardEnumerable α] [LT α] [DecidableLT α]
+    [LawfulUpwardEnumerable α] [LawfulUpwardEnumerableLT α] [Monad n] [LawfulMonad n]
+    {γ LargeEnough hl upperBound} {next hn} {acc} (Pl wf f) :
+    loop γ Pl LargeEnough hl upperBound acc next hn f =
+      loop.wf (α := α) (n := n) γ Pl wf LargeEnough hl upperBound acc next hn f := by
+  haveI : Nonempty γ := ⟨acc⟩
+  rw [loop, WellFounded.extrinsicFix₃_eq_fix]; rotate_left
+  · exact InvImage.wf _ wf
+  · fun_induction loop.wf γ Pl wf LargeEnough hl upperBound acc  next hn f
+    · rw [WellFounded.fix_eq]
+      simp only [↓reduceDIte, *]
+      apply bind_congr; intro forInStep
+      split
+      · simp only
+        split
+        · simp_all
+        · simp
+      · simp
+    · rw [WellFounded.fix_eq]
+      simp_all
+
+private theorem Iterator.instIteratorLoop.loopWf_eq [UpwardEnumerable α] [LT α] [DecidableLT α]
+    [LawfulUpwardEnumerable α] [LawfulUpwardEnumerableLT α]
+    {n : Type u → Type w} [Monad n] [LawfulMonad n] (γ : Type u)
+    {lift} [instLawfulMonadLiftFunction : Std.Internal.LawfulMonadLiftBindFunction (m := Id) (n := n) lift]
+    (Pl : α → γ → ForInStep γ → Prop)
+    (wf : IteratorLoop.WellFounded (Rxo.Iterator α) Id Pl)
+    (LargeEnough : α → Prop) (hl : ∀ a b : α, UpwardEnumerable.LE a b → LargeEnough a → LargeEnough b)
+    (upperBound : α) (acc : γ) (next : α) (h : LargeEnough next)
+    (f : (out : α) → LargeEnough out → out < upperBound → (c : γ) → n (Subtype (fun s : ForInStep γ => Pl out c s))) :
+    loop.wf γ Pl wf LargeEnough hl upperBound acc next h f = (do
       if hu : next < upperBound then
-        loop γ upperBound next init (fun a ha₁ ha₂ c => f a ?hf c) next ?hle hu
-      else
-        return init
-    | ⟨⟨none, _⟩⟩ => return init
-  where
-    @[specialize]
-    loop γ (upperBound : α) least acc
-        (f : (out : α) → UpwardEnumerable.LE least out → out < upperBound → (c : γ) → n (ForInStep γ))
-        (next : α) (hl : UpwardEnumerable.LE least next) (hu : next < upperBound) : n γ := do
-      match ← f next hl hu acc with
-      | .yield acc' =>
-        match hs : succ? next with
-        | some next' =>
-          if hu : next' < upperBound then
-            loop γ upperBound least acc' f next' ?hle' hu
-          else
-            return acc'
-        | none => return acc'
-      | .done acc' => return acc'
-  finally
-    case hf =>
-      rw [Monadic.isPlausibleIndirectOutput_iff]
-      obtain ⟨n, hn⟩ := ha₁
-      exact ⟨n, hn, ha₂⟩
-    case hle =>
-      exact UpwardEnumerable.le_refl _
-    case hle' =>
-      refine UpwardEnumerable.le_trans hl ⟨1, ?_⟩
-      simp [succMany?_one, hs]
-
-theorem Iterator.instIteratorLoop.loop_eq [UpwardEnumerable α] [LT α] [DecidableLT α]
-    [LawfulUpwardEnumerable α] [LawfulUpwardEnumerableLT α]
-    {n : Type u → Type w} [Monad n] [LawfulMonad n] {γ : Type u}
-    {lift} [Internal.LawfulMonadLiftBindFunction lift]
-    {PlausibleForInStep} {upperBound} {next} {hl} {hu} {f} {acc} {wf} :
-    loop (α := α) (n := n) γ PlausibleForInStep wf upperBound least acc f next hl hu =
-      (do
-        match ← f next hl hu acc with
-        | ⟨.yield c, _⟩ =>
+        match ← f next h hu acc with
+        | ⟨.yield acc', _⟩ =>
           letI it' : IterM (α := Rxo.Iterator α) Id α := ⟨⟨succ? next, upperBound⟩⟩
-          IterM.DefaultConsumers.forIn' (m := Id) lift γ
-            PlausibleForInStep wf it' c it'.IsPlausibleIndirectOutput (fun _ => id)
-            (fun b h c => f b
-                (by
-                  refine UpwardEnumerable.le_trans hl ?_
-                  simp only [Monadic.isPlausibleIndirectOutput_iff, it',
-                    ← succMany?_add_one_eq_succ?_bind_succMany?] at h
-                  exact ⟨h.choose + 1, h.choose_spec.1⟩)
-                (by
-                  simp only [Monadic.isPlausibleIndirectOutput_iff, it'] at h
-                  exact h.choose_spec.2) c)
-        | ⟨.done c, _⟩ => return c) := by
-  rw [loop]
-  apply bind_congr
-  intro step
+          IterM.DefaultConsumers.forIn' (m := Id) (n := n) lift γ Pl it' acc'
+            it'.IsPlausibleIndirectOutput (fun _ => id)
+            fun next' h acc' => f next'
+              (by
+                refine hl next next' ?_ ‹_›
+                simp only [it', Monadic.isPlausibleIndirectOutput_iff,
+                  ← succMany?_add_one_eq_succ?_bind_succMany?] at h
+                exact ⟨h.choose + 1, h.choose_spec.1⟩)
+              (by
+                simp only [it', Monadic.isPlausibleIndirectOutput_iff] at h
+                exact h.choose_spec.2)
+              acc'
+        | ⟨.done acc', _⟩ => return acc'
+      else return acc) := by
+  haveI : Nonempty γ := ⟨acc⟩
+  rw [loop.wf]
+  congr 1; ext hu
+  apply bind_congr; intro forInStep
   split
   · split
-    · split
-      · simp only [*]
-        rw [IterM.DefaultConsumers.forIn']
-        simp only [Monadic.step_eq_step, Monadic.step, ↓reduceIte, *,
-          Internal.LawfulMonadLiftBindFunction.liftBind_pure]
-        rw [loop_eq (lift := lift), Shrink.inflate_deflate]
-        apply bind_congr
-        intro step
+    · rw [loopWf_eq (lift := lift) _ Pl wf]
+      rw [IterM.DefaultConsumers.forIn'_eq_match_step (lift := lift) Pl wf]; rotate_left
+      · simp only [Monadic.step_eq_step, Monadic.step,
+          Shrink.inflate_deflate, instLawfulMonadLiftFunction.liftBind_pure, *]
         split
-        · apply IterM.DefaultConsumers.forIn'_eq_forIn'
-          intros; rfl
+        · apply bind_congr; intro forInStep
+          split
+          · apply IterM.DefaultConsumers.forIn'_eq_forIn' Pl wf <;> (intros; rfl)
+          · simp
         · simp
-      · simp only [*]
-        rw [IterM.DefaultConsumers.forIn']
-        simp [Monadic.step_eq_step, Monadic.step, *,
-          Internal.LawfulMonadLiftBindFunction.liftBind_pure]
-    · simp only [*]
-      rw [IterM.DefaultConsumers.forIn']
-      simp [Monadic.step_eq_step, Monadic.step, Internal.LawfulMonadLiftBindFunction.liftBind_pure]
+    · rw [IterM.DefaultConsumers.forIn'_eq_match_step Pl wf]
+      simp [Monadic.step_eq_step, Monadic.step, instLawfulMonadLiftFunction.liftBind_pure, *]
   · simp
 termination_by IteratorLoop.WithWF.mk ⟨⟨some next, upperBound⟩⟩ acc (hwf := wf)
 decreasing_by
-      simp [IteratorLoop.rel, Monadic.isPlausibleStep_iff, Monadic.step, *]
+  simp [IteratorLoop.rel, Monadic.isPlausibleStep_iff, Monadic.step, *]
 
 instance Iterator.instLawfulIteratorLoop [UpwardEnumerable α] [LT α] [DecidableLT α]
     [LawfulUpwardEnumerable α] [LawfulUpwardEnumerableLT α]
     {n : Type u → Type w} [Monad n] [LawfulMonad n] :
     LawfulIteratorLoop (Rxo.Iterator α) Id n where
   lawful := by
-    intro lift instLawfulMonadLiftFunction
-    ext γ PlausibleForInStep hwf it init f
-    simp only [IteratorLoop.forIn, IteratorLoop.defaultImplementation]
-    rw [IterM.DefaultConsumers.forIn']
-    simp only [Monadic.step_eq_step, Monadic.step]
-    simp only [Internal.LawfulMonadLiftBindFunction.liftBind_pure]
+    intro lift instLawfulMonadLiftFunction γ it init Pl wf f
+    simp only [IteratorLoop.defaultImplementation, IteratorLoop.forIn,
+      IterM.DefaultConsumers.forIn'_eq_wf Pl wf]
+    rw [IterM.DefaultConsumers.forIn'.wf]
+    split; rotate_left
+    · simp [Monadic.step_eq_step, Monadic.step, Internal.LawfulMonadLiftBindFunction.liftBind_pure]
+    rename_i next _
+    rw [instIteratorLoop.loop_eq_wf Pl wf, instIteratorLoop.loopWf_eq (lift := lift)]
+    simp only [Monadic.step_eq_step, Monadic.step, instLawfulMonadLiftFunction.liftBind_pure,
+      Shrink.inflate_deflate]
     split
-    · rename_i it f next upperBound f'
-      simp
+    · apply bind_congr; intro forInStep
       split
       · simp only
-        rw [instIteratorLoop.loop_eq (lift := lift)]
-        apply bind_congr
-        intro step
-        split
-        · apply IterM.DefaultConsumers.forIn'_eq_forIn'
-          intro b c hPb hQb
-          congr
-        · simp
+        rw [← IterM.DefaultConsumers.forIn'_eq_wf Pl wf _]
+        apply IterM.DefaultConsumers.forIn'_eq_forIn' Pl wf <;> all_goals (intros; rfl)
       · simp
     · simp
 
@@ -1265,10 +1269,6 @@ instance Iterator.instIteratorCollect [UpwardEnumerable α]
     {n : Type u → Type w} [Monad n] : IteratorCollect (Rxi.Iterator α) Id n :=
   .defaultImplementation
 
-instance Iterator.instIteratorCollectPartial [UpwardEnumerable α]
-    {n : Type u → Type w} [Monad n] : IteratorCollectPartial (Rxi.Iterator α) Id n :=
-  .defaultImplementation
-
 theorem Iterator.Monadic.isPlausibleOutput_next {a} [UpwardEnumerable α]
     {it : IterM (α := Rxi.Iterator α) Id α} (h : it.internalState.next = some a) :
     it.IsPlausibleOutput a := by
@@ -1508,148 +1508,151 @@ section IteratorLoop
 
 /--
 An efficient {name}`IteratorLoop` instance:
-As long as the compiler cannot optimize away the {name}`Option` in the internal state, we use a
-special loop implementation.
+As long as the compiler cannot optimize away the {name}`Option` in the internal state, we use a special
+loop implementation.
 -/
 @[always_inline, inline]
-instance Iterator.instIteratorLoop [UpwardEnumerable α]
-    [LawfulUpwardEnumerable α]
+instance Iterator.instIteratorLoop [UpwardEnumerable α] [LawfulUpwardEnumerable α]
     {n : Type u → Type w} [Monad n] :
     IteratorLoop (Rxi.Iterator α) Id n where
-  forIn _ γ Pl wf it init f :=
+  forIn _ γ Pl it init f :=
     match it with
     | ⟨⟨some next⟩⟩ =>
-        loop γ Pl wf next init (fun a ha c => f a ?hf c) next ?hle
+      loop γ Pl (UpwardEnumerable.LE next ·) (fun a b hab hna => ?hle) init next ?hle'' (fun a ha c => f a ?hf c)
     | ⟨⟨none⟩⟩ => return init
   where
-    @[specialize]
-    loop γ Pl wf least acc
-        (f : (out : α) → UpwardEnumerable.LE least out → (c : γ) → n (Subtype (fun s : ForInStep γ => Pl out c s)))
-        (next : α) (hl : UpwardEnumerable.LE least next) : n γ := do
-      match ← f next hl acc with
-      | ⟨.yield acc', _⟩ =>
-        match hs : UpwardEnumerable.succ? next with
-        | some next' =>
-            loop γ Pl wf least acc' f next' ?hle'
-        | none => return acc'
-      | ⟨.done acc', _⟩ => return acc'
-    termination_by IteratorLoop.WithWF.mk ⟨⟨some next⟩⟩ acc (hwf := wf)
-    decreasing_by
-      simp [IteratorLoop.rel, Monadic.isPlausibleStep_iff,
-        Monadic.step, *]
+    @[always_inline, inline]
+    loop γ (Pl : α → γ → ForInStep γ → Prop) (LargeEnough : α → Prop) (hl : ∀ a b : α, UpwardEnumerable.LE a b → LargeEnough a → LargeEnough b)
+        (acc : γ) (next : α) (h : LargeEnough next)
+        (f : (out : α) → LargeEnough out → (c : γ) → n (Subtype (Pl out c))) : n γ :=
+      haveI : Nonempty γ := ⟨acc⟩
+      WellFounded.extrinsicFix₃ (C₃ := fun _ _ _ => n γ) (InvImage (IteratorLoop.rel _ Id Pl) (fun x => (⟨Rxi.Iterator.mk (some x.1)⟩, x.2.1)))
+        (fun next acc (h : LargeEnough next) G => do
+          match ← f next h acc with
+          | ⟨.yield acc', h'⟩ =>
+            match hs : UpwardEnumerable.succ? next with
+            | some next' => G next' acc' (hl _ _ ?hle' h) ?decreasing
+            | none => return acc'
+          | ⟨.done acc', _⟩ => return acc') next acc h
   finally
     case hf =>
       rw [Monadic.isPlausibleIndirectOutput_iff]
       exact ha
     case hle =>
-      exact UpwardEnumerable.le_refl _
+      exact UpwardEnumerable.le_trans hna hab
     case hle' =>
-      refine UpwardEnumerable.le_trans hl ⟨1, ?_⟩
-      simp [succMany?_one, hs]
-
-/--
-An efficient {name}`IteratorLoopPartial` instance:
-As long as the compiler cannot optimize away the {name}`Option` in the internal state, we use a
-special loop implementation.
--/
-partial instance Iterator.instIteratorLoopPartial [UpwardEnumerable α]
-    [LawfulUpwardEnumerable α]
-    {n : Type u → Type w} [Monad n] : IteratorLoopPartial (Rxi.Iterator α) Id n where
-  forInPartial _ γ it init f :=
-    match it with
-    | ⟨⟨some next⟩⟩ => loop γ next init (fun a ha c => f a ?hf c) next ?hle
-    | ⟨⟨none⟩⟩ => return init
-  where
-    @[specialize]
-    loop γ least acc
-        (f : (out : α) → UpwardEnumerable.LE least out → (c : γ) → n (ForInStep γ))
-        (next : α) (hl : UpwardEnumerable.LE least next) : n γ := do
-      match ← f next hl acc with
-      | .yield acc' =>
-        match hs : succ? next with
-        | some next' =>
-            loop γ least acc' f next' ?hle'
-        | none => return acc'
-      | .done acc' => return acc'
-  finally
-    case hf =>
-      rw [Monadic.isPlausibleIndirectOutput_iff]
-      exact ha
-    case hle =>
+      refine ⟨1, ?_⟩
+      simpa [succMany?_one] using hs
+    case hle'' =>
       exact UpwardEnumerable.le_refl _
-    case hle' =>
-      refine UpwardEnumerable.le_trans hl ⟨1, ?_⟩
-      simp [succMany?_one, hs]
+    case decreasing =>
+      simp_wf; simp [IteratorLoop.rel, Monadic.isPlausibleStep_iff, Monadic.step, *]
 
-theorem Iterator.instIteratorLoop.loop_eq [UpwardEnumerable α]
+private noncomputable def Iterator.instIteratorLoop.loop.wf [UpwardEnumerable α]
     [LawfulUpwardEnumerable α]
-    {n : Type u → Type w} [Monad n] [LawfulMonad n] {γ : Type u}
-    {lift} [Internal.LawfulMonadLiftBindFunction lift]
-    {PlausibleForInStep next hl f acc wf} :
-    loop (α := α) (n := n) γ PlausibleForInStep wf least acc f next hl =
-      (do
-        match ← f next hl acc with
-        | ⟨.yield c, _⟩ =>
+    {n : Type u → Type w} [Monad n] (γ : Type u)
+    (Pl : α → γ → ForInStep γ → Prop)
+    (wf : IteratorLoop.WellFounded (Rxi.Iterator α) Id Pl)
+    (LargeEnough : α → Prop) (hl : ∀ a b : α, UpwardEnumerable.LE a b → LargeEnough a → LargeEnough b)
+    (acc : γ) (next : α) (h : LargeEnough next)
+    (f : (out : α) → LargeEnough out → (c : γ) → n (Subtype (fun s : ForInStep γ => Pl out c s))) :
+    n γ := do
+    match ← f next h acc with
+    | ⟨.yield acc', _⟩ =>
+      match hs : UpwardEnumerable.succ? next with
+      | some next' =>
+        loop.wf γ Pl wf LargeEnough hl acc' next' (hl _ _ ?hle h) f
+      | none => return acc'
+    | ⟨.done acc', _⟩ => return acc'
+termination_by IteratorLoop.WithWF.mk ⟨⟨some next⟩⟩ acc (hwf := wf)
+decreasing_by
+  simp [IteratorLoop.rel, Monadic.isPlausibleStep_iff, Monadic.step, *]
+where finally
+  case hle =>
+    refine ⟨1, ?_⟩
+    simpa [succMany?_one] using hs
+
+private theorem Iterator.instIteratorLoop.loop_eq_wf [UpwardEnumerable α]
+    [LawfulUpwardEnumerable α] [Monad n] [LawfulMonad n]
+    {γ LargeEnough hl} {next hn} {acc} (Pl wf f) :
+    loop γ Pl LargeEnough hl acc next hn f =
+      loop.wf (α := α) (n := n) γ Pl wf LargeEnough hl acc next hn f := by
+  haveI : Nonempty γ := ⟨acc⟩
+  rw [loop, WellFounded.extrinsicFix₃_eq_fix]; rotate_left
+  · exact InvImage.wf _ wf
+  · fun_induction loop.wf γ Pl wf LargeEnough hl acc  next hn f
+    · rw [WellFounded.fix_eq]
+      apply bind_congr; intro forInStep
+      split
+      · simp only
+        split
+        · simp_all
+        · simp
+      · simp
+
+private theorem Iterator.instIteratorLoop.loopWf_eq [UpwardEnumerable α]
+    [LawfulUpwardEnumerable α]
+    {n : Type u → Type w} [Monad n] [LawfulMonad n] (γ : Type u)
+    {lift} [instLawfulMonadLiftFunction : Std.Internal.LawfulMonadLiftBindFunction (m := Id) (n := n) lift]
+    (Pl : α → γ → ForInStep γ → Prop)
+    (wf : IteratorLoop.WellFounded (Rxi.Iterator α) Id Pl)
+    (LargeEnough : α → Prop) (hl : ∀ a b : α, UpwardEnumerable.LE a b → LargeEnough a → LargeEnough b)
+    (acc : γ) (next : α) (h : LargeEnough next)
+    (f : (out : α) → LargeEnough out → (c : γ) → n (Subtype (fun s : ForInStep γ => Pl out c s))) :
+    loop.wf γ Pl wf LargeEnough hl acc next h f = (do
+        match ← f next h acc with
+        | ⟨.yield acc', _⟩ =>
           letI it' : IterM (α := Rxi.Iterator α) Id α := ⟨⟨succ? next⟩⟩
-          IterM.DefaultConsumers.forIn' (m := Id) lift γ
-            PlausibleForInStep wf it' c it'.IsPlausibleIndirectOutput (fun _ => id)
-            (fun b h c => f b
-                (by
-                  refine UpwardEnumerable.le_trans hl ?_
-                  simp only [Monadic.isPlausibleIndirectOutput_iff, it',
-                    ← succMany?_add_one_eq_succ?_bind_succMany?] at h
-                  exact ⟨h.choose + 1, h.choose_spec⟩)
-                c)
-        | ⟨.done c, _⟩ => return c) := by
-  rw [loop]
-  apply bind_congr
-  intro step
+          IterM.DefaultConsumers.forIn' (m := Id) (n := n) lift γ Pl it' acc'
+            it'.IsPlausibleIndirectOutput (fun _ => id)
+            fun next' h acc' => f next'
+              (by
+                refine hl next next' ?_ ‹_›
+                simp only [it', Monadic.isPlausibleIndirectOutput_iff,
+                  ← succMany?_add_one_eq_succ?_bind_succMany?] at h
+                exact ⟨h.choose + 1, h.choose_spec⟩)
+              acc'
+        | ⟨.done acc', _⟩ => return acc') := by
+  haveI : Nonempty γ := ⟨acc⟩
+  rw [loop.wf]
+  apply bind_congr; intro forInStep
   split
   · split
-    · split
-      · rename_i heq
-        cases heq
-        simp only [*]
-        rw [IterM.DefaultConsumers.forIn']
-        simp only [Monadic.step_eq_step, Monadic.step, *,
-          Internal.LawfulMonadLiftBindFunction.liftBind_pure]
-        rw [loop_eq (lift := lift), Shrink.inflate_deflate]
-        apply bind_congr
-        intro step
+    · rw [loopWf_eq (lift := lift) _ Pl wf]
+      rw [IterM.DefaultConsumers.forIn'_eq_match_step (lift := lift) Pl wf]; rotate_left
+      · simp only [Monadic.step_eq_step, Monadic.step,
+          Shrink.inflate_deflate, instLawfulMonadLiftFunction.liftBind_pure, *]
+        apply bind_congr; intro forInStep
         split
-        · apply IterM.DefaultConsumers.forIn'_eq_forIn'
-          intros; rfl
+        · apply IterM.DefaultConsumers.forIn'_eq_forIn' Pl wf <;> (intros; rfl)
         · simp
-      · rename_i heq
-        cases heq
-    · simp only [*]
-      rw [IterM.DefaultConsumers.forIn']
-      simp [Monadic.step_eq_step, Monadic.step, Internal.LawfulMonadLiftBindFunction.liftBind_pure]
+    · rw [IterM.DefaultConsumers.forIn'_eq_match_step Pl wf]
+      simp [Monadic.step_eq_step, Monadic.step, instLawfulMonadLiftFunction.liftBind_pure, *]
   · simp
 termination_by IteratorLoop.WithWF.mk ⟨⟨some next⟩⟩ acc (hwf := wf)
 decreasing_by
-      simp [IteratorLoop.rel, Monadic.isPlausibleStep_iff, Monadic.step, *]
+  simp [IteratorLoop.rel, Monadic.isPlausibleStep_iff, Monadic.step, *]
 
 instance Iterator.instLawfulIteratorLoop [UpwardEnumerable α]
-    [LawfulUpwardEnumerable α] {n : Type u → Type w} [Monad n] [LawfulMonad n] :
+    [LawfulUpwardEnumerable α]
+    {n : Type u → Type w} [Monad n] [LawfulMonad n] :
     LawfulIteratorLoop (Rxi.Iterator α) Id n where
   lawful := by
-    intro lift instLawfulMonadLiftFunction
-    ext γ PlausibleForInStep hwf it init f
-    simp only [IteratorLoop.forIn, IteratorLoop.defaultImplementation]
-    rw [IterM.DefaultConsumers.forIn']
-    simp only [Monadic.step_eq_step, Monadic.step]
-    simp only [Internal.LawfulMonadLiftBindFunction.liftBind_pure]
+    intro lift instLawfulMonadLiftFunction γ it init Pl wf f
+    simp only [IteratorLoop.defaultImplementation, IteratorLoop.forIn,
+      IterM.DefaultConsumers.forIn'_eq_wf Pl wf]
+    rw [IterM.DefaultConsumers.forIn'.wf]
+    split; rotate_left
+    · simp [Monadic.step_eq_step, Monadic.step, Internal.LawfulMonadLiftBindFunction.liftBind_pure]
+    rename_i next _
+    rw [instIteratorLoop.loop_eq_wf Pl wf, instIteratorLoop.loopWf_eq (lift := lift)]
+    simp only [Monadic.step_eq_step, Monadic.step, instLawfulMonadLiftFunction.liftBind_pure,
+      Shrink.inflate_deflate]
+    apply bind_congr; intro forInStep
     split
-    · rename_i it f next upperBound f'
-      rw [instIteratorLoop.loop_eq (lift := lift), Shrink.inflate_deflate]
-      apply bind_congr
-      intro step
-      split
-      · apply IterM.DefaultConsumers.forIn'_eq_forIn'
-        intro b c hPb hQb
-        congr
-      · simp
+    · simp only
+      rw [← IterM.DefaultConsumers.forIn'_eq_wf Pl wf _]
+      apply IterM.DefaultConsumers.forIn'_eq_forIn' Pl wf <;> all_goals (intros; rfl)
     · simp
 
 end IteratorLoop

src/Init/Data/Range/Polymorphic/UpwardEnumerable.lean

@@ -318,7 +318,7 @@ This function uses an `UpwardEnumerable α` instance.
 
 If no other implementation is provided in UpwardEnumerable instance, succMany? repeatedly applies succ?.
 -/
-@[always_inline, inline]
+@[always_inline, inline, expose]
 def UpwardEnumerable.succMany {α : Type u} [UpwardEnumerable α]
     [LawfulUpwardEnumerable α] [InfinitelyUpwardEnumerable α]
     (n : Nat) (a : α) :=

src/Init/Data/SInt/Basic.lean

@@ -150,7 +150,7 @@ Converts an 8-bit signed integer to an arbitrary-precision integer that denotes
 
 This function is overridden at runtime with an efficient implementation.
 -/
-@[extern "lean_int8_to_int"]
+@[extern "lean_int8_to_int", tagged_return]
 def Int8.toInt (i : Int8) : Int := i.toBitVec.toInt
 /--
 Converts an 8-bit signed integer to a natural number, mapping all negative numbers to `0`.
@@ -503,7 +503,7 @@ Converts a 16-bit signed integer to an arbitrary-precision integer that denotes
 
 This function is overridden at runtime with an efficient implementation.
 -/
-@[extern "lean_int16_to_int"]
+@[extern "lean_int16_to_int", tagged_return]
 def Int16.toInt (i : Int16) : Int := i.toBitVec.toInt
 /--
 Converts a 16-bit signed integer to a natural number, mapping all negative numbers to `0`.

src/Init/Data/Slice/Array/Iterator.lean

@@ -60,9 +60,7 @@ instance SubarrayIterator.instFinite : Finite (SubarrayIterator α) Id :=
   .of_finitenessRelation instFinitelessRelation
 
 instance [Monad m] : IteratorCollect (SubarrayIterator α) Id m := .defaultImplementation
-instance [Monad m] : IteratorCollectPartial (SubarrayIterator α) Id m := .defaultImplementation
 instance [Monad m] : IteratorLoop (SubarrayIterator α) Id m := .defaultImplementation
-instance [Monad m] : IteratorLoopPartial (SubarrayIterator α) Id m := .defaultImplementation
 
 @[inline, expose]
 def Subarray.instToIterator :=

src/Init/Data/Slice/Notation.lean

@@ -9,6 +9,7 @@ prelude
 public import Init.Data.Range.Polymorphic.PRange
 
 set_option doc.verso true
+set_option linter.missingDocs true
 
 public section
 
@@ -30,6 +31,9 @@ This typeclass indicates how to obtain slices of elements of {lit}`α` over rang
 The type of the resulting slices is {lit}`γ`.
 -/
 class Rcc.Sliceable (α : Type u) (β : outParam (Type v)) (γ : outParam (Type w)) where
+  /--
+  Slices {name}`carrier` from {lean}`range.lower` to {lean}`range.upper`, both inclusive.
+  -/
   mkSlice (carrier : α) (range : Rcc β) : γ
 
 /--
@@ -39,6 +43,9 @@ This typeclass indicates how to obtain slices of elements of {lit}`α` over rang
 The type of resulting the slices is {lit}`γ`.
 -/
 class Rco.Sliceable (α : Type u) (β : outParam (Type v)) (γ : outParam (Type w)) where
+  /--
+  Slices {name}`carrier` from {lean}`range.lower` (inclusive) to {lean}`range.upper` (exclusive).
+  -/
   mkSlice (carrier : α) (range : Rco β) : γ
 
 /--
@@ -48,6 +55,9 @@ This typeclass indicates how to obtain slices of elements of {lit}`α` over rang
 The type of the resulting slices is {lit}`γ`.
 -/
 class Rci.Sliceable (α : Type u) (β : outParam (Type v)) (γ : outParam (Type w)) where
+  /--
+  Slices {name}`carrier` from {lean}`range.lower` (inclusive).
+  -/
   mkSlice (carrier : α) (range : Rci β) : γ
 
 /--
@@ -57,6 +67,9 @@ This typeclass indicates how to obtain slices of elements of {lit}`α` over rang
 The type of the resulting slices is {lit}`γ`.
 -/
 class Roc.Sliceable (α : Type u) (β : outParam (Type v)) (γ : outParam (Type w)) where
+  /--
+  Slices {name}`carrier` from {lean}`range.lower` (exclusive) to {lean}`range.upper` (inclusive).
+  -/
   mkSlice (carrier : α) (range : Roc β) : γ
 
 /--
@@ -66,6 +79,9 @@ This typeclass indicates how to obtain slices of elements of {lit}`α` over rang
 The type of the resulting slices is {lit}`γ`.
 -/
 class Roo.Sliceable (α : Type u) (β : outParam (Type v)) (γ : outParam (Type w)) where
+  /--
+  Slices {name}`carrier` from {lean}`range.lower` to {lean}`range.upper`, both exclusive.
+  -/
   mkSlice (carrier : α) (range : Roo β) : γ
 
 /--
@@ -75,6 +91,9 @@ This typeclass indicates how to obtain slices of elements of {lit}`α` over rang
 The type of the resulting slices is {lit}`γ`.
 -/
 class Roi.Sliceable (α : Type u) (β : outParam (Type v)) (γ : outParam (Type w)) where
+  /--
+  Slices {name}`carrier` from {lean}`range.lower` (exclusive).
+  -/
   mkSlice (carrier : α) (range : Roi β) : γ
 
 /--
@@ -84,6 +103,9 @@ This typeclass indicates how to obtain slices of elements of {lit}`α` over rang
 The type of the resulting slices is {lit}`γ`.
 -/
 class Ric.Sliceable (α : Type u) (β : outParam (Type v)) (γ : outParam (Type w)) where
+  /--
+  Slices {name}`carrier` up to {lean}`range.upper` (inclusive).
+  -/
   mkSlice (carrier : α) (range : Ric β) : γ
 
 /--
@@ -93,6 +115,9 @@ This typeclass indicates how to obtain slices of elements of {lit}`α` over rang
 The type of the resulting slices is {lit}`γ`.
 -/
 class Rio.Sliceable (α : Type u) (β : outParam (Type v)) (γ : outParam (Type w)) where
+  /--
+  Slices {name}`carrier` up to {lean}`range.upper` (exclusive).
+  -/
   mkSlice (carrier : α) (range : Rio β) : γ
 
 /--
@@ -102,6 +127,9 @@ index type {lit}`β`.
 The type of the resulting slices is {lit}`γ`.
 -/
 class Rii.Sliceable (α : Type u) (β : outParam (Type v)) (γ : outParam (Type w)) where
+  /--
+  Slices {name}`carrier` with no bounds.
+  -/
   mkSlice (carrier : α) (range : Rii β) : γ
 
 macro_rules

src/Init/Data/String/Basic.lean

@@ -64,103 +64,73 @@ not a sequence of Unicode scalar values.
 -/
 @[inline, expose]
 def ByteArray.utf8Decode? (b : ByteArray) : Option (Array Char) :=
-  go (b.size + 1) 0 #[] (by simp) (by simp)
+  go 0 #[] (by simp)
 where
-  go (fuel : Nat) (i : Nat) (acc : Array Char) (hi : i ≤ b.size) (hf : b.size - i < fuel) : Option (Array Char) :=
-    match fuel, hf with
-    | fuel + 1, _ =>
-      if i = b.size then
-        some acc
-      else
-        match h : utf8DecodeChar? b i with
-        | none => none
-        | some c => go fuel (i + c.utf8Size) (acc.push c)
-            (le_size_of_utf8DecodeChar?_eq_some h)
-            (have := c.utf8Size_pos; have := le_size_of_utf8DecodeChar?_eq_some h; by omega)
-  termination_by structural fuel
+  @[semireducible]
+  go (i : Nat) (acc : Array Char) (hi : i ≤ b.size) : Option (Array Char) :=
+    if i < b.size then
+      match h : utf8DecodeChar? b i with
+      | none => none
+      | some c => go (i + c.utf8Size) (acc.push c) (le_size_of_utf8DecodeChar?_eq_some h)
+    else
+      some acc
+  termination_by b.size - i
+  decreasing_by have := c.utf8Size_pos; omega
 
 @[expose, extern "lean_string_validate_utf8"]
 def ByteArray.validateUTF8 (b : @& ByteArray) : Bool :=
-  go (b.size + 1) 0 (by simp) (by simp)
+  go 0 (by simp)
 where
-  go (fuel : Nat) (i : Nat) (hi : i ≤ b.size) (hf : b.size - i < fuel) : Bool :=
-    match fuel, hf with
-    | fuel + 1, _ =>
-      if hi : i = b.size then
-        true
-      else
-        match h : validateUTF8At b i with
-        | false => false
-        | true => go fuel (i + b[i].utf8ByteSize (isUTF8FirstByte_of_validateUTF8At h))
-            ?_ ?_
-  termination_by structural fuel
+  @[semireducible]
+  go (i : Nat) (hi : i ≤ b.size) : Bool :=
+    if hi : i < b.size then
+      match h : validateUTF8At b i with
+      | false => false
+      | true => go (i + b[i].utf8ByteSize (isUTF8FirstByte_of_validateUTF8At h)) ?_
+    else
+      true
+  termination_by b.size - i
+  decreasing_by
+    have := b[i].utf8ByteSize_pos (isUTF8FirstByte_of_validateUTF8At h); omega
 finally
   all_goals rw [ByteArray.validateUTF8At_eq_isSome_utf8DecodeChar?] at h
   · rw [← ByteArray.utf8Size_utf8DecodeChar (h := h)]
     exact add_utf8Size_utf8DecodeChar_le_size
-  · rw [← ByteArray.utf8Size_utf8DecodeChar (h := h)]
-    have := add_utf8Size_utf8DecodeChar_le_size (h := h)
-    have := (b.utf8DecodeChar i h).utf8Size_pos
-    omega
 
-theorem ByteArray.isSome_utf8Decode?Go_eq_validateUTF8Go {b : ByteArray} {fuel : Nat}
-    {i : Nat} {acc : Array Char} {hi : i ≤ b.size} {hf : b.size - i < fuel} :
-    (utf8Decode?.go b fuel i acc hi hf).isSome = validateUTF8.go b fuel i hi hf := by
+theorem ByteArray.isSome_utf8Decode?Go_eq_validateUTF8Go {b : ByteArray}
+    {i : Nat} {acc : Array Char} {hi : i ≤ b.size} :
+    (utf8Decode?.go b i acc hi).isSome = validateUTF8.go b i hi := by
   fun_induction utf8Decode?.go with
-  | case1 => simp [validateUTF8.go]
-  | case2 i acc hi fuel hf h₁ h₂ =>
-    simp only [Option.isSome_none, validateUTF8.go, h₁, ↓reduceDIte, Bool.false_eq]
+  | case1 i acc hi h₁ h₂ =>
+    unfold validateUTF8.go
+    simp only [Option.isSome_none, ↓reduceDIte, Bool.false_eq, h₁]
     split
     · rfl
     · rename_i heq
       simp [validateUTF8At_eq_isSome_utf8DecodeChar?, h₂] at heq
-  | case3 i acc hi fuel hf h₁ c h₂ ih =>
-    simp [validateUTF8.go, h₁]
+  | case2 i acc hi h₁ c h₂ ih =>
+    unfold validateUTF8.go
+    simp only [↓reduceDIte, ih, h₁]
     split
     · rename_i heq
       simp [validateUTF8At_eq_isSome_utf8DecodeChar?, h₂] at heq
-    · rw [ih]
-      congr
+    · congr
       rw [← ByteArray.utf8Size_utf8DecodeChar (h := by simp [h₂])]
       simp [utf8DecodeChar, h₂]
+  | case3 => unfold validateUTF8.go; simp [*]
 
 theorem ByteArray.isSome_utf8Decode?_eq_validateUTF8 {b : ByteArray} :
     b.utf8Decode?.isSome = b.validateUTF8 :=
   b.isSome_utf8Decode?Go_eq_validateUTF8Go
 
-theorem ByteArray.utf8Decode?.go.congr {b b' : ByteArray} {fuel fuel' i i' : Nat} {acc acc' : Array Char} {hi hi' hf hf'}
-    (hbb' : b = b') (hii' : i = i') (hacc : acc = acc') :
-    ByteArray.utf8Decode?.go b fuel i acc hi hf = ByteArray.utf8Decode?.go b' fuel' i' acc' hi' hf' := by
-  subst hbb' hii' hacc
-  fun_induction ByteArray.utf8Decode?.go b fuel i acc hi hf generalizing fuel' with
-  | case1 =>
-    rw [go.eq_def]
-    split
-    simp
-  | case2 =>
-    rw [go.eq_def]
-    split <;> split
-    · simp_all
-    · split <;> simp_all
-  | case3 =>
-    conv => rhs; rw [go.eq_def]
-    split <;> split
-    · simp_all
-    · split
-      · simp_all
-      · rename_i c₁ hc₁ ih _ _ _ _ _ c₂ hc₂
-        obtain rfl : c₁ = c₂ := by rw [← Option.some_inj, ← hc₁, ← hc₂]
-        apply ih
-
 @[simp]
 theorem ByteArray.utf8Decode?_empty : ByteArray.empty.utf8Decode? = some #[] := by
   simp [utf8Decode?, utf8Decode?.go]
 
-private theorem ByteArray.isSome_utf8Decode?go_iff {b : ByteArray} {fuel i : Nat} {hi : i ≤ b.size} {hf} {acc : Array Char} :
-    (ByteArray.utf8Decode?.go b fuel i acc hi hf).isSome ↔ IsValidUTF8 (b.extract i b.size) := by
+private theorem ByteArray.isSome_utf8Decode?go_iff {b : ByteArray} {hi : i ≤ b.size} {acc : Array Char} :
+    (ByteArray.utf8Decode?.go b i acc hi).isSome ↔ IsValidUTF8 (b.extract i b.size) := by
   fun_induction ByteArray.utf8Decode?.go with
-  | case1 => simp
-  | case2 fuel i hi hf acc h₁ h₂ =>
+  | case1 i hi acc h₁ h₂ =>
     simp only [Option.isSome_none, Bool.false_eq_true, false_iff]
     rintro ⟨l, hl⟩
     have : l ≠ [] := by
@@ -170,7 +140,7 @@ private theorem ByteArray.isSome_utf8Decode?go_iff {b : ByteArray} {fuel i : Nat
     rw [← l.cons_head_tail this] at hl
     rw [utf8DecodeChar?_eq_utf8DecodeChar?_extract, hl, List.utf8DecodeChar?_utf8Encode_cons] at h₂
     simp at h₂
-  | case3 i acc hi fuel hf h₁ c h₂ ih =>
+  | case2 i acc hi h₁ c h₂ ih =>
     rw [ih]
     have h₂' := h₂
     rw [utf8DecodeChar?_eq_utf8DecodeChar?_extract] at h₂'
@@ -179,6 +149,9 @@ private theorem ByteArray.isSome_utf8Decode?go_iff {b : ByteArray} {fuel i : Nat
       (le_size_of_utf8DecodeChar?_eq_some h₂)] at hl ⊢
     rw [ByteArray.append_inj_left hl (by have := le_size_of_utf8DecodeChar?_eq_some h₂; simp; omega),
       ← List.utf8Encode_singleton, isValidUTF8_utf8Encode_singleton_append_iff]
+  | case3 i =>
+    have : i = b.size  := by omega
+    simp [*]
 
 theorem ByteArray.isSome_utf8Decode?_iff {b : ByteArray} :
     b.utf8Decode?.isSome ↔ IsValidUTF8 b := by
@@ -291,7 +264,7 @@ Examples:
 * `"abc".length = 3`
 * `"L∃∀N".length = 4`
 -/
-@[extern "lean_string_length", expose]
+@[extern "lean_string_length", expose, tagged_return]
 def String.length (b : @& String) : Nat :=
   b.toList.length
 
@@ -305,27 +278,21 @@ theorem String.length_toList {s : String} : s.toList.length = s.length := (rfl)
 @[deprecated String.length_toList (since := "2025-10-30")]
 theorem String.length_data {b : String} : b.toList.length = b.length := (rfl)
 
-private theorem ByteArray.utf8Decode?go_eq_utf8Decode?go_extract {b : ByteArray} {fuel i : Nat} {hi : i ≤ b.size} {hf} {acc : Array Char} :
-    utf8Decode?.go b fuel i acc hi hf = (utf8Decode?.go (b.extract i b.size) fuel 0 #[] (by simp) (by simp [hf])).map (acc ++ ·) := by
-  fun_cases utf8Decode?.go b fuel i acc hi hf with
-  | case1 =>
-      rw [utf8Decode?.go]
-      simp only [size_extract, Nat.le_refl, Nat.min_eq_left, Nat.zero_add, List.push_toArray,
-        List.nil_append]
-      rw [if_pos (by omega)]
-      simp
-  | case2 fuel hf₁ h₁ h₂ hf₂ =>
+private theorem ByteArray.utf8Decode?go_eq_utf8Decode?go_extract {b : ByteArray} {hi : i ≤ b.size} {acc : Array Char} :
+    utf8Decode?.go b i acc hi = (utf8Decode?.go (b.extract i b.size) 0 #[] (by simp)).map (acc ++ ·) := by
+  fun_cases utf8Decode?.go b i acc hi with
+  | case1 h₁ h₂ =>
     rw [utf8Decode?.go]
     simp only [size_extract, Nat.le_refl, Nat.min_eq_left, Nat.zero_add, List.push_toArray,
       List.nil_append]
-    rw [if_neg (by omega)]
+    rw [if_pos (by omega)]
     rw [utf8DecodeChar?_eq_utf8DecodeChar?_extract] at h₂
     split <;> simp_all
-  | case3 fuel hf₁ h₁ c h₂ hf₂ =>
+  | case2 h₁ c h₂ =>
     conv => rhs; rw [utf8Decode?.go]
     simp only [size_extract, Nat.le_refl, Nat.min_eq_left, Nat.zero_add, List.push_toArray,
       List.nil_append]
-    rw [if_neg (by omega)]
+    rw [if_pos (by omega)]
     rw [utf8DecodeChar?_eq_utf8DecodeChar?_extract] at h₂
     split
     · simp_all
@@ -338,20 +305,25 @@ private theorem ByteArray.utf8Decode?go_eq_utf8Decode?go_extract {b : ByteArray}
       simp only [size_extract, Nat.le_refl, Nat.min_eq_left, Option.map_map, ByteArray.extract_extract]
       have : (fun x => acc ++ x) ∘ (fun x => #[c] ++ x) = fun x => acc.push c ++ x := by funext; simp
       simp [(by omega : i + (b.size - i) = b.size), this]
-termination_by fuel
+  | case3 =>
+      rw [utf8Decode?.go]
+      simp only [size_extract, Nat.le_refl, Nat.min_eq_left, Nat.zero_add, List.push_toArray,
+        List.nil_append]
+      rw [if_neg (by omega)]
+      simp
+termination_by b.size - i
 
 theorem ByteArray.utf8Decode?_utf8Encode_singleton_append {l : ByteArray} {c : Char} :
     ([c].utf8Encode ++ l).utf8Decode? = l.utf8Decode?.map (#[c] ++ ·) := by
   rw [utf8Decode?, utf8Decode?.go,
-    if_neg (by simp [List.utf8Encode_singleton]; have := c.utf8Size_pos; omega)]
+    if_pos (by simp [List.utf8Encode_singleton]; have := c.utf8Size_pos; omega)]
   split
   · simp_all [List.utf8DecodeChar?_utf8Encode_singleton_append]
   · rename_i d h
     obtain rfl : c = d := by simpa [List.utf8DecodeChar?_utf8Encode_singleton_append] using h
     rw [utf8Decode?go_eq_utf8Decode?go_extract, utf8Decode?]
     simp only [List.push_toArray, List.nil_append, Nat.zero_add]
-    congr 1
-    apply ByteArray.utf8Decode?.go.congr _ rfl rfl
+    congr 2
     apply extract_append_eq_right _ (by simp)
     simp [List.utf8Encode_singleton]
 
@@ -824,16 +796,25 @@ The region of `s` from `b` (inclusive) to `e` (exclusive) is copied to a newly-a
 
 If `b`'s offset is greater than or equal to that of `e`, then the resulting string is `""`.
 
+If possible, prefer `String.slice`, which avoids the allocation.
 -/
 @[extern "lean_string_utf8_extract"]
-def Pos.extract {s : @& String} (b e : @& s.Pos) : String where
+def extract {s : @& String} (b e : @& s.Pos) : String where
   toByteArray := s.toByteArray.extract b.offset.byteIdx e.offset.byteIdx
   isValidUTF8 := b.isValidUTF8_extract e
 
+@[deprecated String.extract (since := "2025-12-01")]
+def Pos.extract {s : String} (b e : @& s.Pos) : String :=
+  s.extract b e
+
+@[simp]
+theorem toByteArray_extract {s : String} {b e : s.Pos} :
+    (s.extract b e).toByteArray = s.toByteArray.extract b.offset.byteIdx e.offset.byteIdx := (rfl)
+
 /-- Creates a `String` from a `String.Slice` by copying the bytes. -/
 @[inline]
 def Slice.copy (s : Slice) : String :=
-  s.startInclusive.extract s.endExclusive
+  s.str.extract s.startInclusive s.endExclusive
 
 theorem Slice.toByteArray_copy {s : Slice} :
     s.copy.toByteArray = s.str.toByteArray.extract s.startInclusive.offset.byteIdx s.endExclusive.offset.byteIdx := (rfl)
@@ -1196,7 +1177,7 @@ position. -/
 def Slice.Pos.get? {s : Slice} (pos : s.Pos) : Option Char :=
   if h : pos = s.endPos then none else some (pos.get h)
 
-/-- Returns the byte at the given position in the string, or panicks if the position is the end
+/-- Returns the byte at the given position in the string, or panics if the position is the end
 position. -/
 @[expose]
 def Slice.Pos.get! {s : Slice} (pos : s.Pos) : Char :=
@@ -1387,54 +1368,58 @@ theorem Pos.offset_ofCopy {s : Slice} {pos : s.copy.Pos} : pos.ofCopy.offset = p
 
 /-- Given a slice `s` and a position on `s`, obtain the corresponding position on `s.copy.` -/
 @[inline]
-def Slice.Pos.toCopy {s : Slice} (pos : s.Pos) : s.copy.Pos where
+def Slice.Pos.copy {s : Slice} (pos : s.Pos) : s.copy.Pos where
   offset := pos.offset
   isValid := Pos.Raw.isValid_copy_iff.2 pos.isValidForSlice
 
-@[simp]
-theorem Slice.Pos.offset_toCopy {s : Slice} {pos : s.Pos} : pos.toCopy.offset = pos.offset := (rfl)
+@[deprecated Slice.Pos.copy (since := "2025-12-01")]
+def Slice.Pos.toCopy {s : Slice} (pos : s.Pos) : s.copy.Pos :=
+  pos.copy
 
 @[simp]
-theorem Slice.Pos.ofCopy_toCopy {s : Slice} {pos : s.Pos} : pos.toCopy.ofCopy = pos :=
+theorem Slice.Pos.offset_copy {s : Slice} {pos : s.Pos} : pos.copy.offset = pos.offset := (rfl)
+
+@[simp]
+theorem Slice.Pos.ofCopy_copy {s : Slice} {pos : s.Pos} : pos.copy.ofCopy = pos :=
   Slice.Pos.ext (by simp)
 
 @[simp]
-theorem Pos.toCopy_ofCopy {s : Slice} {pos : s.copy.Pos} : pos.ofCopy.toCopy = pos :=
+theorem Pos.copy_ofCopy {s : Slice} {pos : s.copy.Pos} : pos.ofCopy.copy = pos :=
   Pos.ext (by simp)
 
 theorem Pos.ofCopy_inj {s : Slice} {pos pos' : s.copy.Pos} : pos.ofCopy = pos'.ofCopy ↔ pos = pos' :=
-  ⟨fun h => by simpa using congrArg Slice.Pos.toCopy h, (· ▸ rfl)⟩
+  ⟨fun h => by simpa using congrArg Slice.Pos.copy h, (· ▸ rfl)⟩
 
 @[simp]
-theorem Slice.startPos_copy {s : Slice} : s.copy.startPos = s.startPos.toCopy :=
+theorem Slice.startPos_copy {s : Slice} : s.copy.startPos = s.startPos.copy :=
   String.Pos.ext (by simp)
 
 @[simp]
-theorem Slice.endPos_copy {s : Slice} : s.copy.endPos = s.endPos.toCopy :=
+theorem Slice.endPos_copy {s : Slice} : s.copy.endPos = s.endPos.copy :=
   String.Pos.ext (by simp)
 
-theorem Slice.Pos.get_toCopy {s : Slice} {pos : s.Pos} (h) :
-    pos.toCopy.get h = pos.get (by rintro rfl; simp at h) := by
+theorem Slice.Pos.get_copy {s : Slice} {pos : s.Pos} (h) :
+    pos.copy.get h = pos.get (by rintro rfl; simp at h) := by
   rw [String.Pos.get, Slice.Pos.get_eq_utf8DecodeChar, Slice.Pos.get_eq_utf8DecodeChar]
-  simp only [str_toSlice, toByteArray_copy, startInclusive_toSlice, startPos_copy, offset_toCopy,
+  simp only [str_toSlice, toByteArray_copy, startInclusive_toSlice, startPos_copy, offset_copy,
     Slice.offset_startPos, Pos.Raw.byteIdx_zero, Pos.offset_toSlice, Nat.zero_add]
   rw [ByteArray.utf8DecodeChar_eq_utf8DecodeChar_extract]
   conv => lhs; congr; rw [ByteArray.extract_extract]
   conv => rhs; rw [ByteArray.utf8DecodeChar_eq_utf8DecodeChar_extract]
   exact ByteArray.utf8DecodeChar_extract_congr _ _ _
 
-theorem Slice.Pos.get_eq_get_toCopy {s : Slice} {pos : s.Pos} {h} :
-    pos.get h = pos.toCopy.get (ne_of_apply_ne Pos.ofCopy (by simp [h])) :=
-  (get_toCopy _).symm
+theorem Slice.Pos.get_eq_get_copy {s : Slice} {pos : s.Pos} {h} :
+    pos.get h = pos.copy.get (ne_of_apply_ne Pos.ofCopy (by simp [h])) :=
+  (get_copy _).symm
 
-theorem Slice.Pos.byte_toCopy {s : Slice} {pos : s.Pos} (h) :
-    pos.toCopy.byte h = pos.byte (by rintro rfl; simp at h) := by
+theorem Slice.Pos.byte_copy {s : Slice} {pos : s.Pos} (h) :
+    pos.copy.byte h = pos.byte (by rintro rfl; simp at h) := by
   rw [String.Pos.byte, Slice.Pos.byte, Slice.Pos.byte]
   simp [getUTF8Byte, String.getUTF8Byte, toByteArray_copy, ByteArray.getElem_extract]
 
-theorem Slice.Pos.byte_eq_byte_toCopy {s : Slice} {pos : s.Pos} {h} :
-    pos.byte h = pos.toCopy.byte (ne_of_apply_ne Pos.ofCopy (by simp [h])) :=
-  (byte_toCopy _).symm
+theorem Slice.Pos.byte_eq_byte_copy {s : Slice} {pos : s.Pos} {h} :
+    pos.byte h = pos.copy.byte (ne_of_apply_ne Pos.ofCopy (by simp [h])) :=
+  (byte_copy _).symm
 
 /-- Given a position in `s.sliceFrom p₀`, obtain the corresponding position in `s`. -/
 @[inline]
@@ -1521,7 +1506,7 @@ theorem Slice.Pos.copy_eq_append_get {s : Slice} {pos : s.Pos} (h : pos ≠ s.en
     ∃ t₁ t₂ : String, s.copy = t₁ ++ singleton (pos.get h) ++ t₂ ∧ t₁.utf8ByteSize = pos.offset.byteIdx := by
   obtain ⟨t₂, ht₂⟩ := (s.sliceFrom pos).copy.eq_singleton_append (by simpa [← Pos.ofCopy_inj, ← ofSliceFrom_inj])
   refine ⟨(s.sliceTo pos).copy, t₂, ?_, by simp⟩
-  simp only [Slice.startPos_copy, get_toCopy, get_eq_get_ofSliceFrom, ofSliceFrom_startPos] at ht₂
+  simp only [Slice.startPos_copy, get_copy, get_eq_get_ofSliceFrom, ofSliceFrom_startPos] at ht₂
   rw [append_assoc, ← ht₂, ← copy_eq_copy_sliceTo]
 
 theorem Slice.Pos.utf8ByteSize_byte {s : Slice} {pos : s.Pos} {h : pos ≠ s.endPos} :
@@ -1667,7 +1652,7 @@ def Slice.pos! (s : Slice) (off : String.Pos.Raw) : s.Pos :=
 
 /-- Advances a valid position on a string to the next valid position, given a proof that the
 position is not the past-the-end position, which guarantees that such a position exists. -/
-@[expose, extern "lean_string_utf8_next_fast"]
+@[expose, extern "lean_string_utf8_next_fast", tagged_return]
 def Pos.next {s : @& String} (pos : @& s.Pos) (h : pos ≠ s.endPos) : s.Pos :=
   ofToSlice (Slice.Pos.next pos.toSlice (ne_of_apply_ne Pos.ofToSlice (by simpa)))
 
@@ -1751,8 +1736,8 @@ theorem Pos.offset_cast {s t : String} {pos : s.Pos} {h : s = t} :
 theorem Pos.cast_rfl {s : String} {pos : s.Pos} : pos.cast rfl = pos :=
   Pos.ext (by simp)
 
-theorem Pos.toCopy_toSlice_eq_cast {s : String} (p : s.Pos) :
-    p.toSlice.toCopy = p.cast copy_toSlice.symm :=
+theorem Pos.copy_toSlice_eq_cast {s : String} (p : s.Pos) :
+    p.toSlice.copy = p.cast copy_toSlice.symm :=
   Pos.ext (by simp)
 
 /-- Given a byte position within a string slice, obtains the smallest valid position that is
@@ -2435,6 +2420,35 @@ def Pos.slice! {s : String} (pos : s.Pos) (p₀ p₁ : s.Pos) :
     (s.slice! p₀ p₁).Pos :=
   Slice.Pos.slice! pos.toSlice _ _
 
+theorem extract_eq_copy_slice {s : String} (p₀ p₁ : s.Pos) (h : p₀ ≤ p₁) :
+    s.extract p₀ p₁ = (s.slice p₀ p₁ h).copy := by
+  simp [← toByteArray_inj, Slice.toByteArray_copy]
+
+/--
+Copies a region of a slice to a new string.
+
+The region of `s` from `b` (inclusive) to `e` (exclusive) is copied to a newly-allocated `String`.
+
+If `b`'s offset is greater than or equal to that of `e`, then the resulting string is `""`.
+
+If possible, prefer `Slice.slice`, which avoids the allocation.
+-/
+@[inline]
+def Slice.extract (s : Slice) (p₀ p₁ : s.Pos) : String :=
+  s.str.extract p₀.str p₁.str
+
+@[simp]
+theorem Slice.Pos.str_le_str_iff {s : Slice} {p q : s.Pos} : p.str ≤ q.str ↔ p ≤ q := by
+  simp [String.Pos.le_iff, Slice.Pos.le_iff, Pos.Raw.le_iff]
+
+@[simp]
+theorem Slice.Pos.str_lt_str_iff {s : Slice} {p q : s.Pos} : p.str < q.str ↔ p < q := by
+  simp [String.Pos.lt_iff, Slice.Pos.lt_iff, Pos.Raw.lt_iff]
+
+theorem Slice.extract_eq_copy_slice {s : Slice} (p₀ p₁ : s.Pos) (h : p₀ ≤ p₁) :
+    s.extract p₀ p₁ = (s.slice p₀ p₁ h).copy := by
+  simp [← toByteArray_inj, Slice.toByteArray_copy, Slice.extract]
+
 /--
 Advances the position `p` `n` times.
 
@@ -2642,7 +2656,7 @@ This is a legacy function. The recommended alternative is `String.Pos.next`, com
 Example:
 * `let abc := "abc"; abc.get (abc.next' 0 (by decide)) = 'b'`
 -/
-@[extern "lean_string_utf8_next_fast", expose]
+@[extern "lean_string_utf8_next_fast", expose, tagged_return]
 def Pos.Raw.next' (s : @& String) (p : @& Pos.Raw) (h : ¬ p.atEnd s) : Pos.Raw :=
   let c := p.get s
   p + c
@@ -2734,10 +2748,6 @@ where
     | [],    _, _ => []
     | c::cs, i, e => if i = e then [] else c :: go₂ cs (i + c) e
 
-@[extern "lean_string_utf8_extract", expose, deprecated Pos.Raw.extract (since := "2025-10-14")]
-def extract : (@& String) → (@& Pos.Raw) → (@& Pos.Raw) → String
-  | s, b, e => Pos.Raw.extract s b e
-
 def Pos.Raw.offsetOfPosAux (s : String) (pos : Pos.Raw) (i : Pos.Raw) (offset : Nat) : Nat :=
   if i >= pos then offset
   else if h : i.atEnd s then

src/Init/Data/String/Decode.lean

@@ -529,7 +529,7 @@ public def assemble₂ (w x : UInt8) : Option Char :=
   else
     let r := assemble₂Unchecked w x
     if r < 0x80 then
-      none -- overlong encodinlg
+      none -- overlong encoding
     else
       some ⟨r, ?onemore⟩
 where finally
@@ -1441,6 +1441,9 @@ public def utf8ByteSize (c : UInt8) (_h : c.IsUTF8FirstByte) : Nat :=
   else
     4
 
+public theorem utf8ByteSize_pos (c : UInt8) (h : c.IsUTF8FirstByte) : 0 < c.utf8ByteSize h := by
+  fun_cases utf8ByteSize <;> simp
+
 def _root_.ByteArray.utf8DecodeChar?.FirstByte.utf8ByteSize : FirstByte → Nat
   | .invalid => 0
   | .done => 1

src/Init/Data/String/Lemmas/Basic.lean

@@ -40,7 +40,7 @@ theorem singleton_ne_empty {c : Char} : singleton c ≠ "" := by
   simp [singleton]
 
 @[simp]
-theorem Slice.Pos.toCopy_inj {s : Slice} {p₁ p₂ : s.Pos} : p₁.toCopy = p₂.toCopy ↔ p₁ = p₂ := by
+theorem Slice.Pos.copy_inj {s : Slice} {p₁ p₂ : s.Pos} : p₁.copy = p₂.copy ↔ p₁ = p₂ := by
   simp [String.Pos.ext_iff, Pos.ext_iff]
 
 @[simp]

src/Init/Data/String/Lemmas/Search.lean

@@ -16,7 +16,6 @@ open String.Slice Pattern
 
 variable {ρ : Type} {σ : Slice → Type}
 variable [∀ s, Std.Iterators.Iterator (σ s) Id (SearchStep s)]
-variable [∀ s, Std.Iterators.Finite (σ s) Id]
 variable [∀ s, Std.Iterators.IteratorLoop (σ s) Id Id]
 
 @[simp]

src/Init/Data/String/Lemmas/Splits.lean

@@ -56,25 +56,25 @@ theorem Slice.Pos.Splits.cast {s₁ s₂ : Slice} {p : s₁.Pos} {t₁ t₂ : St
     p.Splits t₁ t₂ → (p.cast h).Splits t₁ t₂ :=
   splits_cast_iff.mpr
 
-theorem Slice.Pos.Splits.toCopy {s : Slice} {p : s.Pos} {t₁ t₂ : String}
-    (h : p.Splits t₁ t₂) : p.toCopy.Splits t₁ t₂ where
+theorem Slice.Pos.Splits.copy {s : Slice} {p : s.Pos} {t₁ t₂ : String}
+    (h : p.Splits t₁ t₂) : p.copy.Splits t₁ t₂ where
   eq_append := h.eq_append
   offset_eq_rawEndPos := by simpa using h.offset_eq_rawEndPos
 
-theorem Slice.Pos.splits_of_splits_toCopy {s : Slice} {p : s.Pos} {t₁ t₂ : String}
-    (h : p.toCopy.Splits t₁ t₂) : p.Splits t₁ t₂ where
+theorem Slice.Pos.splits_of_splits_copy {s : Slice} {p : s.Pos} {t₁ t₂ : String}
+    (h : p.copy.Splits t₁ t₂) : p.Splits t₁ t₂ where
   eq_append := h.eq_append
   offset_eq_rawEndPos := by simpa using h.offset_eq_rawEndPos
 
 @[simp]
-theorem Slice.Pos.splits_toCopy_iff {s : Slice} {p : s.Pos} {t₁ t₂ : String} :
-    p.toCopy.Splits t₁ t₂ ↔ p.Splits t₁ t₂ :=
-  ⟨splits_of_splits_toCopy, (·.toCopy)⟩
+theorem Slice.Pos.splits_copy_iff {s : Slice} {p : s.Pos} {t₁ t₂ : String} :
+    p.copy.Splits t₁ t₂ ↔ p.Splits t₁ t₂ :=
+  ⟨splits_of_splits_copy, (·.copy)⟩
 
 @[simp]
 theorem Pos.splits_toSlice_iff {s : String} {p : s.Pos} {t₁ t₂ : String} :
     p.toSlice.Splits t₁ t₂ ↔ p.Splits t₁ t₂ := by
-  rw [← Slice.Pos.splits_toCopy_iff, p.toCopy_toSlice_eq_cast, splits_cast_iff]
+  rw [← Slice.Pos.splits_copy_iff, p.copy_toSlice_eq_cast, splits_cast_iff]
 
 theorem Pos.Splits.toSlice {s : String} {p : s.Pos} {t₁ t₂ : String}
     (h : p.Splits t₁ t₂) : p.toSlice.Splits t₁ t₂ :=
@@ -112,15 +112,15 @@ theorem Pos.Splits.eq {s : String} {p : s.Pos} {t₁ t₂ t₃ t₄}
 
 theorem Slice.Pos.Splits.eq_left {s : Slice} {p : s.Pos} {t₁ t₂ t₃ t₄}
     (h₁ : p.Splits t₁ t₂) (h₂ : p.Splits t₃ t₄) : t₁ = t₃ :=
-  (splits_toCopy_iff.2 h₁).eq_left (splits_toCopy_iff.2 h₂)
+  (splits_copy_iff.2 h₁).eq_left (splits_copy_iff.2 h₂)
 
 theorem Slice.Pos.Splits.eq_right {s : Slice} {p : s.Pos} {t₁ t₂ t₃ t₄}
     (h₁ : p.Splits t₁ t₂) (h₂ : p.Splits t₃ t₄) : t₂ = t₄ :=
-  (splits_toCopy_iff.2 h₁).eq_right (splits_toCopy_iff.2 h₂)
+  (splits_copy_iff.2 h₁).eq_right (splits_copy_iff.2 h₂)
 
 theorem Slice.Pos.Splits.eq {s : Slice} {p : s.Pos} {t₁ t₂ t₃ t₄}
     (h₁ : p.Splits t₁ t₂) (h₂ : p.Splits t₃ t₄) : t₁ = t₃ ∧ t₂ = t₄ :=
-  (splits_toCopy_iff.2 h₁).eq (splits_toCopy_iff.2 h₂)
+  (splits_copy_iff.2 h₁).eq (splits_copy_iff.2 h₂)
 
 @[simp]
 theorem splits_endPos (s : String) : s.endPos.Splits s "" where
@@ -161,11 +161,11 @@ theorem Slice.splits_endPos (s : Slice) : s.endPos.Splits s.copy "" where
 @[simp]
 theorem Slice.splits_endPos_iff {s : Slice} :
     s.endPos.Splits t₁ t₂ ↔ t₁ = s.copy ∧ t₂ = "" := by
-  rw [← Pos.splits_toCopy_iff, ← endPos_copy, String.splits_endPos_iff]
+  rw [← Pos.splits_copy_iff, ← endPos_copy, String.splits_endPos_iff]
 
 theorem Slice.Pos.Splits.eq_endPos_iff {s : Slice} {p : s.Pos} (h : p.Splits t₁ t₂) :
     p = s.endPos ↔ t₂ = "" := by
-  rw [← toCopy_inj, ← endPos_copy, h.toCopy.eq_endPos_iff]
+  rw [← copy_inj, ← endPos_copy, h.copy.eq_endPos_iff]
 
 @[simp]
 theorem Slice.splits_startPos (s : Slice) : s.startPos.Splits "" s.copy where
@@ -175,11 +175,11 @@ theorem Slice.splits_startPos (s : Slice) : s.startPos.Splits "" s.copy where
 @[simp]
 theorem Slice.splits_startPos_iff {s : Slice} :
     s.startPos.Splits t₁ t₂ ↔ t₁ = "" ∧ t₂ = s.copy := by
-  rw [← Pos.splits_toCopy_iff, ← startPos_copy, String.splits_startPos_iff]
+  rw [← Pos.splits_copy_iff, ← startPos_copy, String.splits_startPos_iff]
 
 theorem Slice.Pos.Splits.eq_startPos_iff {s : Slice} {p : s.Pos} (h : p.Splits t₁ t₂) :
     p = s.startPos ↔ t₁ = "" := by
-  rw [← toCopy_inj, ← startPos_copy, h.toCopy.eq_startPos_iff]
+  rw [← copy_inj, ← startPos_copy, h.copy.eq_startPos_iff]
 
 theorem Pos.splits_next_right {s : String} (p : s.Pos) (hp : p ≠ s.endPos) :
     p.Splits (s.sliceTo p).copy (singleton (p.get hp) ++ (s.sliceFrom (p.next hp)).copy) where

src/Init/Data/String/Pattern/Pred.lean

@@ -79,9 +79,11 @@ instance : Std.Iterators.Finite (ForwardCharPredSearcher p s) Id :=
 instance : Std.Iterators.IteratorLoop (ForwardCharPredSearcher p s) Id Id :=
   .defaultImplementation
 
+@[default_instance]
 instance {p : Char → Bool} : ToForwardSearcher p (ForwardCharPredSearcher p) where
   toSearcher := iter p
 
+@[default_instance]
 instance {p : Char → Bool} : ForwardPattern p := .defaultImplementation
 
 instance {p : Char → Prop} [DecidablePred p] : ToForwardSearcher p (ForwardCharPredSearcher p) where
@@ -153,9 +155,11 @@ instance : Std.Iterators.Finite (BackwardCharPredSearcher s) Id :=
 instance : Std.Iterators.IteratorLoop (BackwardCharPredSearcher s) Id Id :=
   .defaultImplementation
 
+@[default_instance]
 instance {p : Char → Bool} : ToBackwardSearcher p BackwardCharPredSearcher where
   toSearcher := iter p
 
+@[default_instance]
 instance {p : Char → Bool} : BackwardPattern p := ToBackwardSearcher.defaultImplementation
 
 instance {p : Char → Prop} [DecidablePred p] : ToBackwardSearcher p BackwardCharPredSearcher where

src/Init/Data/String/Slice.lean

@@ -188,15 +188,9 @@ instance [Std.Iterators.Finite (σ s) Id] : Std.Iterators.Finite (SplitIterator
 instance [Monad n] : Std.Iterators.IteratorCollect (SplitIterator pat s) Id n :=
   .defaultImplementation
 
-instance [Monad n] : Std.Iterators.IteratorCollectPartial (SplitIterator pat s) Id n :=
-  .defaultImplementation
-
 instance [Monad n] : Std.Iterators.IteratorLoop (SplitIterator pat s) Id n :=
   .defaultImplementation
 
-instance [Monad n] : Std.Iterators.IteratorLoopPartial (SplitIterator pat s) Id n :=
-  .defaultImplementation
-
 end SplitIterator
 
 /--
@@ -290,18 +284,10 @@ instance [Monad n] {s} :
     Std.Iterators.IteratorCollect (SplitInclusiveIterator pat s) Id n :=
   .defaultImplementation
 
-instance [Monad n] {s} :
-    Std.Iterators.IteratorCollectPartial (SplitInclusiveIterator pat s) Id n :=
-  .defaultImplementation
-
 instance [Monad n] {s} :
     Std.Iterators.IteratorLoop (SplitInclusiveIterator pat s) Id n :=
   .defaultImplementation
 
-instance [Monad n] {s} :
-    Std.Iterators.IteratorLoopPartial (SplitInclusiveIterator pat s) Id n :=
-  .defaultImplementation
-
 end SplitInclusiveIterator
 
 /--
@@ -640,16 +626,9 @@ instance [Std.Iterators.Finite (σ s) Id] : Std.Iterators.Finite (RevSplitIterat
 instance [Monad m] [Monad n] : Std.Iterators.IteratorCollect (RevSplitIterator ρ s) m n :=
   .defaultImplementation
 
-instance [Monad m] [Monad n] :
-    Std.Iterators.IteratorCollectPartial (RevSplitIterator ρ s) m n :=
-  .defaultImplementation
-
 instance [Monad m] [Monad n] : Std.Iterators.IteratorLoop (RevSplitIterator ρ s) m n :=
   .defaultImplementation
 
-instance [Monad m] [Monad n] : Std.Iterators.IteratorLoopPartial (RevSplitIterator ρ s) m n :=
-  .defaultImplementation
-
 end RevSplitIterator
 
 /--
@@ -921,15 +900,9 @@ instance [Pure m] : Std.Iterators.Finite (PosIterator s) m :=
 instance [Monad m] [Monad n] : Std.Iterators.IteratorCollect (PosIterator s) m n :=
   .defaultImplementation
 
-instance [Monad m] [Monad n] : Std.Iterators.IteratorCollectPartial (PosIterator s) m n :=
-  .defaultImplementation
-
 instance [Monad m] [Monad n] : Std.Iterators.IteratorLoop (PosIterator s) m n :=
   .defaultImplementation
 
-instance [Monad m] [Monad n] : Std.Iterators.IteratorLoopPartial (PosIterator s) m n :=
-  .defaultImplementation
-
 docs_to_verso positions
 
 end PosIterator
@@ -1011,16 +984,9 @@ instance [Pure m] : Std.Iterators.Finite (RevPosIterator s) m :=
 instance [Monad m] [Monad n] : Std.Iterators.IteratorCollect (RevPosIterator s) m n :=
   .defaultImplementation
 
-instance [Monad m] [Monad n] :
-    Std.Iterators.IteratorCollectPartial (RevPosIterator s) m n :=
-  .defaultImplementation
-
 instance [Monad m] [Monad n] : Std.Iterators.IteratorLoop (RevPosIterator s) m n :=
   .defaultImplementation
 
-instance [Monad m] [Monad n] : Std.Iterators.IteratorLoopPartial (RevPosIterator s) m n :=
-  .defaultImplementation
-
 docs_to_verso revPositions
 
 end RevPosIterator
@@ -1098,15 +1064,9 @@ instance [Pure m] : Std.Iterators.Finite ByteIterator m :=
 instance [Monad m] [Monad n] : Std.Iterators.IteratorCollect ByteIterator m n :=
   .defaultImplementation
 
-instance [Monad m] [Monad n] : Std.Iterators.IteratorCollectPartial ByteIterator m n :=
-  .defaultImplementation
-
 instance [Monad m] [Monad n] : Std.Iterators.IteratorLoop ByteIterator m n :=
   .defaultImplementation
 
-instance [Monad m] [Monad n] : Std.Iterators.IteratorLoopPartial ByteIterator m n :=
-  .defaultImplementation
-
 docs_to_verso bytes
 
 end ByteIterator
@@ -1185,15 +1145,9 @@ instance [Pure m] : Std.Iterators.Finite RevByteIterator m :=
 instance [Monad m] [Monad n] : Std.Iterators.IteratorCollect RevByteIterator m n :=
   .defaultImplementation
 
-instance [Monad m] [Monad n] : Std.Iterators.IteratorCollectPartial RevByteIterator m n :=
-  .defaultImplementation
-
 instance [Monad m] [Monad n] : Std.Iterators.IteratorLoop RevByteIterator m n :=
   .defaultImplementation
 
-instance [Monad m] [Monad n] : Std.Iterators.IteratorLoopPartial RevByteIterator m n :=
-  .defaultImplementation
-
 docs_to_verso revBytes
 
 end RevByteIterator
@@ -1249,7 +1203,8 @@ def foldr {α : Type u} (f : Char → α → α) (init : α) (s : Slice) : α :=
 Checks whether the slice can be interpreted as the decimal representation of a natural number.
 
 A slice can be interpreted as a decimal natural number if it is not empty and all the characters in
-it are digits.
+it are digits. Underscores ({lit}`_`) are allowed as digit separators for readability, but cannot appear
+at the start, at the end, or consecutively.
 
 Use {name (scope := "Init.Data.String.Slice")}`toNat?` or
 {name (scope := "Init.Data.String.Slice")}`toNat!` to convert such a slice to a natural number.
@@ -1260,21 +1215,39 @@ Examples:
  * {lean}`"5".toSlice.isNat = true`
  * {lean}`"05".toSlice.isNat = true`
  * {lean}`"587".toSlice.isNat = true`
+ * {lean}`"1_000".toSlice.isNat = true`
+ * {lean}`"100_000_000".toSlice.isNat = true`
  * {lean}`"-587".toSlice.isNat = false`
  * {lean}`" 5".toSlice.isNat = false`
  * {lean}`"2+3".toSlice.isNat = false`
  * {lean}`"0xff".toSlice.isNat = false`
+ * {lean}`"_123".toSlice.isNat = false`
+ * {lean}`"123_".toSlice.isNat = false`
+ * {lean}`"12__34".toSlice.isNat = false`
 -/
 @[inline]
 def isNat (s : Slice) : Bool :=
-  !s.isEmpty && s.all Char.isDigit
+  if s.isEmpty then
+    false
+  else
+    -- Track: isFirst, lastWasUnderscore, lastCharWasDigit, valid
+    let result := s.foldl (fun (isFirst, lastWasUnderscore, _lastCharWasDigit, valid) c =>
+      let isDigit := c.isDigit
+      let isUnderscore := c = '_'
+      let newValid := valid && (isDigit || isUnderscore) &&
+                      !(isFirst && isUnderscore) &&  -- Cannot start with underscore
+                      !(lastWasUnderscore && isUnderscore)  -- No consecutive underscores
+      (false, isUnderscore, isDigit, newValid))
+      (true, false, false, true)
+    -- Must be valid and last character must have been a digit (not underscore)
+    result.2.2.2 && result.2.2.1
 
 /--
 Interprets a slice as the decimal representation of a natural number, returning it. Returns
 {name}`none` if the slice does not contain a decimal natural number.
 
 A slice can be interpreted as a decimal natural number if it is not empty and all the characters in
-it are digits.
+it are digits. Underscores ({lit}`_`) are allowed as digit separators and are ignored during parsing.
 
 Use {name}`isNat` to check whether {name}`toNat?` would return {name}`some`.
 {name (scope := "Init.Data.String.Slice")}`toNat!` is an alternative that panics instead of
@@ -1285,6 +1258,8 @@ Examples:
  * {lean}`"0".toSlice.toNat? = some 0`
  * {lean}`"5".toSlice.toNat? = some 5`
  * {lean}`"587".toSlice.toNat? = some 587`
+ * {lean}`"1_000".toSlice.toNat? = some 1000`
+ * {lean}`"100_000_000".toSlice.toNat? = some 100000000`
  * {lean}`"-587".toSlice.toNat? = none`
  * {lean}`" 5".toSlice.toNat? = none`
  * {lean}`"2+3".toSlice.toNat? = none`
@@ -1292,7 +1267,7 @@ Examples:
 -/
 def toNat? (s : Slice) : Option Nat :=
   if s.isNat then
-    some <| s.foldl (fun n c => n * 10 + (c.toNat - '0'.toNat)) 0
+    some <| s.foldl (fun n c => if c = '_' then n else n * 10 + (c.toNat - '0'.toNat)) 0
   else
     none
 
@@ -1301,7 +1276,7 @@ Interprets a slice as the decimal representation of a natural number, returning
 slice does not contain a decimal natural number.
 
 A slice can be interpreted as a decimal natural number if it is not empty and all the characters in
-it are digits.
+it are digits. Underscores ({lit}`_`) are allowed as digit separators and are ignored during parsing.
 
 Use {name}`isNat` to check whether {name}`toNat!` would return a value. {name}`toNat?` is a safer
 alternative that returns {name}`none` instead of panicking when the string is not a natural number.
@@ -1310,10 +1285,11 @@ Examples:
  * {lean}`"0".toSlice.toNat! = 0`
  * {lean}`"5".toSlice.toNat! = 5`
  * {lean}`"587".toSlice.toNat! = 587`
+ * {lean}`"1_000".toSlice.toNat! = 1000`
 -/
 def toNat! (s : Slice) : Nat :=
   if s.isNat then
-    s.foldl (fun n c => n * 10 + (c.toNat - '0'.toNat)) 0
+    s.foldl (fun n c => if c = '_' then n else n * 10 + (c.toNat - '0'.toNat)) 0
   else
     panic! "Nat expected"
 

src/Init/Data/String/Substring.lean

@@ -403,25 +403,39 @@ Examples:
 Checks whether the substring can be interpreted as the decimal representation of a natural number.
 
 A substring can be interpreted as a decimal natural number if it is not empty and all the characters
-in it are digits.
+in it are digits. Underscores ({lit}`_`) are allowed as digit separators for readability, but cannot appear
+at the start, at the end, or consecutively.
 
 Use `Substring.toNat?` to convert such a substring to a natural number.
 -/
 @[inline] def isNat (s : Substring.Raw) : Bool :=
-  !s.isEmpty && s.all fun c => c.isDigit
+  if s.isEmpty then
+    false
+  else
+    -- Track: isFirst, lastWasUnderscore, lastCharWasDigit, valid
+    let result := s.foldl (fun (isFirst, lastWasUnderscore, _lastCharWasDigit, valid) c =>
+      let isDigit := c.isDigit
+      let isUnderscore := c = '_'
+      let newValid := valid && (isDigit || isUnderscore) &&
+                      !(isFirst && isUnderscore) &&  -- Cannot start with underscore
+                      !(lastWasUnderscore && isUnderscore)  -- No consecutive underscores
+      (false, isUnderscore, isDigit, newValid))
+      (true, false, false, true)
+    -- Must be valid and last character must have been a digit (not underscore)
+    result.2.2.2 && result.2.2.1
 
 /--
 Checks whether the substring can be interpreted as the decimal representation of a natural number,
 returning the number if it can.
 
 A substring can be interpreted as a decimal natural number if it is not empty and all the characters
-in it are digits.
+in it are digits. Underscores ({lit}`_`) are allowed as digit separators and are ignored during parsing.
 
 Use `Substring.isNat` to check whether the substring is such a substring.
 -/
 def toNat? (s : Substring.Raw) : Option Nat :=
   if s.isNat then
-    some <| s.foldl (fun n c => n*10 + (c.toNat - '0'.toNat)) 0
+    some <| s.foldl (fun n c => if c = '_' then n else n*10 + (c.toNat - '0'.toNat)) 0
   else
     none
 

src/Init/Data/String/Termination.lean

@@ -223,13 +223,13 @@ end Pos
 namespace Slice.Pos
 
 @[simp]
-theorem remainingBytes_toCopy {s : Slice} {p : s.Pos} :
-    p.toCopy.remainingBytes = p.remainingBytes := by
+theorem remainingBytes_copy {s : Slice} {p : s.Pos} :
+    p.copy.remainingBytes = p.remainingBytes := by
   simp [remainingBytes_eq, String.Pos.remainingBytes_eq, Slice.utf8ByteSize_eq]
 
 theorem Splits.remainingBytes_eq {s : Slice} {p : s.Pos} {t₁ t₂} (h : p.Splits t₁ t₂) :
     p.remainingBytes = t₂.utf8ByteSize := by
-  simpa using h.toCopy.remainingBytes_eq
+  simpa using h.copy.remainingBytes_eq
 
 end Slice.Pos
 

src/Init/Data/Sum/Basic.lean

@@ -13,7 +13,7 @@ public section
 /-!
 # Disjoint union of types
 
-This file defines basic operations on the the sum type `α ⊕ β`.
+This file defines basic operations on the sum type `α ⊕ β`.
 
 `α ⊕ β` is the type made of a copy of `α` and a copy of `β`. It is also called *disjoint union*.
 

src/Init/Data/UInt/Basic.lean

@@ -517,7 +517,7 @@ protected def UInt32.shiftRight (a b : UInt32) : UInt32 := ⟨a.toBitVec >>> (UI
 Strict inequality of 32-bit unsigned integers, defined as inequality of the corresponding
 natural numbers. Usually accessed via the `<` operator.
 -/
--- These need to be exposed as `Init.Prelude` already has an instance for bootstrapping puproses and
+-- These need to be exposed as `Init.Prelude` already has an instance for bootstrapping purposes and
 -- they should be defeq
 @[expose] protected def UInt32.lt (a b : UInt32) : Prop := a.toBitVec < b.toBitVec
 /--

src/Init/Data/UInt/BasicAux.lean

@@ -57,7 +57,7 @@ Converts an 8-bit unsigned integer to an arbitrary-precision natural number.
 
 This function is overridden at runtime with an efficient implementation.
 -/
-@[extern "lean_uint8_to_nat"]
+@[extern "lean_uint8_to_nat", tagged_return]
 def UInt8.toNat (n : UInt8) : Nat := n.toBitVec.toNat
 
 instance UInt8.instOfNat : OfNat UInt8 n := ⟨UInt8.ofNat n⟩
@@ -108,7 +108,7 @@ Converts a 16-bit unsigned integer to an arbitrary-precision natural number.
 
 This function is overridden at runtime with an efficient implementation.
 -/
-@[extern "lean_uint16_to_nat"]
+@[extern "lean_uint16_to_nat", tagged_return]
 def UInt16.toNat (n : UInt16) : Nat := n.toBitVec.toNat
 /--
 Converts 16-bit unsigned integers to 8-bit unsigned integers. Wraps around on overflow.

src/Init/GetElem.lean

@@ -240,6 +240,9 @@ theorem getElem_of_getElem? [GetElem? cont idx elem dom] [LawfulGetElem cont idx
     {c : cont} {i : idx} [Decidable (dom c i)] (h : c[i]? = some e) : Exists fun h : dom c i => c[i] = e :=
   getElem?_eq_some_iff.mp h
 
+theorem of_getElem_eq [GetElem? cont idx elem dom] [LawfulGetElem cont idx elem dom]
+    {c : cont} {i : idx} [Decidable (dom c i)] {h} (_ : c[i] = e) : dom c i := h
+
 @[simp] theorem some_getElem_eq_getElem?_iff [GetElem? cont idx elem dom] [LawfulGetElem cont idx elem dom]
     {c : cont} {i : idx} [Decidable (dom c i)] (h : dom c i):
     (some c[i] = c[i]?) ↔ True := by

src/Init/Grind/Config.lean

@@ -202,6 +202,15 @@ structure Config where
   ```
   -/
   funCC := true
+  /--
+  When `true`, use reducible transparency when trying to close goals.
+  In this setting, only declarations marked with `@[reducible]` are unfolded during
+  definitional equality tests.
+
+  This option does not affect the canonicalizer or how theorem patterns are internalized;
+  reducible declarations are always unfolded in those contexts.
+  -/
+  reducible := true
   deriving Inhabited, BEq
 
 /--

src/Init/Grind/Lemmas.lean

@@ -185,4 +185,12 @@ theorem decide_eq_false {p : Prop} {_ : Decidable p} : p = False → decide p =
 theorem of_lookahead (p : Prop) (h : (¬ p) → False) : p = True := by
   simp at h; simp [h]
 
+/-! Nat propagators -/
+
+theorem Nat.and_congr {a b : Nat} {k₁ k₂ k : Nat} (h₁ : a = k₁) (h₂ : b = k₂) : k == k₁ &&& k₂ → a &&& b = k := by simp_all
+theorem Nat.xor_congr {a b : Nat} {k₁ k₂ k : Nat} (h₁ : a = k₁) (h₂ : b = k₂) : k == k₁ ^^^ k₂ → a ^^^ b = k := by simp_all
+theorem Nat.or_congr {a b : Nat} {k₁ k₂ k : Nat} (h₁ : a = k₁) (h₂ : b = k₂) : k == k₁ ||| k₂ → a ||| b = k := by simp_all
+theorem Nat.shiftLeft_congr {a b : Nat} {k₁ k₂ k : Nat} (h₁ : a = k₁) (h₂ : b = k₂) : k == k₁ <<< k₂ → a <<< b = k := by simp_all
+theorem Nat.shiftRight_congr {a b : Nat} {k₁ k₂ k : Nat} (h₁ : a = k₁) (h₂ : b = k₂) : k == k₁ >>> k₂ → a >>> b = k := by simp_all
+
 end Lean.Grind

src/Init/Grind/Module/NatModuleNorm.lean

@@ -79,9 +79,9 @@ theorem Poly.denoteN_append {α} [NatModule α] (ctx : Context α) (p₁ p₂ :
 
 attribute [local simp] Poly.denoteN_append
 
-theorem Poly.denoteN_combine' {α} [NatModule α] (ctx : Context α) (fuel : Nat) (p₁ p₂ : Poly)
-    : p₁.NonnegCoeffs → p₂.NonnegCoeffs → (p₁.combine' fuel p₂).denoteN ctx = p₁.denoteN ctx + p₂.denoteN ctx := by
-  fun_induction p₁.combine' fuel p₂ <;> intro h₁ h₂ <;> try simp [*, zero_add, add_zero]
+theorem Poly.denoteN_combine {α} [NatModule α] (ctx : Context α) (p₁ p₂ : Poly)
+    : p₁.NonnegCoeffs → p₂.NonnegCoeffs → (p₁.combine p₂).denoteN ctx = p₁.denoteN ctx + p₂.denoteN ctx := by
+  fun_induction p₁.combine p₂ <;> intro h₁ h₂ <;> try simp [*, zero_add, add_zero]
   next hx _ h ih =>
     simp at hx
     simp +zetaDelta at h
@@ -103,10 +103,6 @@ theorem Poly.denoteN_combine' {α} [NatModule α] (ctx : Context α) (fuel : Nat
     cases h₂; next h₂ =>
     simp [ih h₁ h₂, *]; ac_rfl
 
-theorem Poly.denoteN_combine {α} [NatModule α] (ctx : Context α) (p₁ p₂ : Poly)
-    : p₁.NonnegCoeffs → p₂.NonnegCoeffs → (p₁.combine p₂).denoteN ctx = p₁.denoteN ctx + p₂.denoteN ctx := by
-  intros; simp [combine, denoteN_combine', *]
-
 theorem Poly.denoteN_mul' {α} [NatModule α] (ctx : Context α) (p : Poly) (k : Nat) : p.NonnegCoeffs → (p.mul' k).denoteN ctx = k • p.denoteN ctx := by
   induction p <;> simp [mul', *, nsmul_zero]
   next ih =>
@@ -151,9 +147,8 @@ theorem Poly.append_Nonneg (p₁ p₂ : Poly) : p₁.NonnegCoeffs → p₂.Nonne
   fun_induction append <;> intro h₁ h₂; simp [*]
   next ih => cases h₁; constructor; assumption; apply ih <;> assumption
 
-theorem Poly.combine'_Nonneg (fuel : Nat) (p₁ p₂ : Poly) : p₁.NonnegCoeffs → p₂.NonnegCoeffs → (p₁.combine' fuel p₂).NonnegCoeffs := by
-  fun_induction Poly.combine'
-  next => apply Poly.append_Nonneg
+theorem Poly.combine_Nonneg (p₁ p₂ : Poly) : p₁.NonnegCoeffs → p₂.NonnegCoeffs → (p₁.combine p₂).NonnegCoeffs := by
+  fun_induction Poly.combine
   next => intros; assumption
   next => intros; assumption
   next ih =>
@@ -172,9 +167,6 @@ theorem Poly.combine'_Nonneg (fuel : Nat) (p₁ p₂ : Poly) : p₁.NonnegCoeffs
     constructor; assumption
     apply ih; constructor; assumption; assumption; assumption
 
-theorem Poly.combine_Nonneg (p₁ p₂ : Poly) : p₁.NonnegCoeffs → p₂.NonnegCoeffs → (p₁.combine p₂).NonnegCoeffs := by
-  simp [combine]; apply Poly.combine'_Nonneg
-
 theorem Expr.toPolyN_Nonneg (e : Expr) : e.toPolyN.NonnegCoeffs := by
   fun_induction toPolyN <;> try constructor <;> simp
   next => constructor; simp; constructor

src/Init/Grind/Ordered/Linarith.lean

@@ -123,26 +123,22 @@ def Poly.append (p₁ p₂ : Poly) : Poly :=
   | .nil => p₂
   | .add k x p₁ => .add k x (append p₁ p₂)
 
-def Poly.combine' (fuel : Nat) (p₁ p₂ : Poly) : Poly :=
-  match fuel with
-  | 0 => p₁.append p₂
-  | fuel + 1 => match p₁, p₂ with
+def Poly.combine (p₁ p₂ : Poly) : Poly :=
+  match p₁, p₂ with
     | .nil, p₂ => p₂
     | p₁, .nil => p₁
     | .add a₁ x₁ p₁, .add a₂ x₂ p₂ =>
       bif Nat.beq x₁ x₂ then
         let a := a₁ + a₂
         bif a == 0 then
-          combine' fuel p₁ p₂
+          combine p₁ p₂
         else
-          .add a x₁ (combine' fuel p₁ p₂)
+          .add a x₁ (combine p₁ p₂)
       else bif Nat.blt x₂ x₁ then
-        .add a₁ x₁ (combine' fuel p₁ (.add a₂ x₂ p₂))
+        .add a₁ x₁ (combine p₁ (.add a₂ x₂ p₂))
       else
-        .add a₂ x₂ (combine' fuel (.add a₁ x₁ p₁) p₂)
-
-def Poly.combine (p₁ p₂ : Poly) : Poly :=
-  combine' 100000000 p₁ p₂
+        .add a₂ x₂ (combine (.add a₁ x₁ p₁) p₂)
+  termination_by sizeOf p₁ + sizeOf p₂
 
 /-- Converts the given expression into a polynomial. -/
 def Expr.toPoly' (e : Expr) : Poly :=
@@ -205,8 +201,8 @@ theorem Poly.denote_append {α} [IntModule α] (ctx : Context α) (p₁ p₂ : P
 
 attribute [local simp] Poly.denote_append
 
-theorem Poly.denote_combine' {α} [IntModule α] (ctx : Context α) (fuel : Nat) (p₁ p₂ : Poly) : (p₁.combine' fuel p₂).denote ctx = p₁.denote ctx + p₂.denote ctx := by
-  fun_induction p₁.combine' fuel p₂ <;>
+theorem Poly.denote_combine {α} [IntModule α] (ctx : Context α) (p₁ p₂ : Poly) : (p₁.combine p₂).denote ctx = p₁.denote ctx + p₂.denote ctx := by
+  fun_induction p₁.combine p₂ <;>
     simp_all +zetaDelta [denote]
   next h _ =>
     rw [Int.add_comm] at h
@@ -214,9 +210,6 @@ theorem Poly.denote_combine' {α} [IntModule α] (ctx : Context α) (fuel : Nat)
   next => rw [add_zsmul]; ac_rfl
   all_goals ac_rfl
 
-theorem Poly.denote_combine {α} [IntModule α] (ctx : Context α) (p₁ p₂ : Poly) : (p₁.combine p₂).denote ctx = p₁.denote ctx + p₂.denote ctx := by
-  simp [combine, denote_combine']
-
 attribute [local simp] Poly.denote_combine
 
 private theorem Expr.denote_toPoly'_go {α} [IntModule α] {k p} (ctx : Context α) (e : Expr)

src/Init/Grind/Ring/CommSolver.lean

@@ -180,6 +180,53 @@ where
         else
           .mult { x := pw₁.x, k := pw₁.k + pw₂.k } (go fuel m₁ m₂)
 
+noncomputable def Mon.mul_k : Mon → Mon → Mon :=
+  Nat.rec
+    (fun m₁ m₂ => concat m₁ m₂)
+    (fun _ ih m₁ m₂ =>
+      Mon.rec (t := m₂)
+        m₁
+        (fun pw₂ m₂' _ => Mon.rec (t := m₁)
+          m₂
+          (fun pw₁ m₁' _ =>
+            Bool.rec (t := pw₁.varLt pw₂)
+              (Bool.rec (t := pw₂.varLt pw₁)
+                (.mult { x := pw₁.x, k := Nat.add pw₁.k pw₂.k } (ih m₁' m₂'))
+                (.mult pw₂ (ih (.mult pw₁ m₁') m₂')))
+              (.mult pw₁ (ih m₁' (.mult pw₂ m₂'))))))
+    hugeFuel
+
+theorem Mon.mul_k_eq_mul : Mon.mul_k m₁ m₂ = Mon.mul m₁ m₂ := by
+  unfold mul_k mul
+  generalize hugeFuel = fuel
+  fun_induction mul.go
+  · rfl
+  · rfl
+  case case3 m₂ _ =>
+    cases m₂
+    · contradiction
+    · dsimp
+  case case4 fuel pw₁ m₁ pw₂ m₂ h ih =>
+    dsimp only
+    rw [h]
+    dsimp only
+    rw [ih]
+  case case5 fuel pw₁ m₁ pw₂ m₂ h₁ h₂ ih =>
+    dsimp only
+    rw [h₁]
+    dsimp only
+    rw [h₂]
+    dsimp only
+    rw [ih]
+  case case6 fuel pw₁ m₁ pw₂ m₂ h₁ h₂ ih =>
+    dsimp only
+    rw [h₁]
+    dsimp only
+    rw [h₂]
+    dsimp only
+    rw [ih]
+    rfl
+
 def Mon.mul_nc (m₁ m₂ : Mon) : Mon :=
   match m₁ with
   | .unit          => m₂
@@ -190,6 +237,28 @@ def Mon.degree : Mon → Nat
   | .unit => 0
   | .mult pw m => pw.k + degree m
 
+noncomputable def Mon.degree_k : Mon → Nat :=
+  Nat.rec
+    (fun m => m.degree)
+    (fun _ ih m =>
+      Mon.rec (t := m)
+        0
+        (fun pw m' _ => Nat.add pw.k (ih m')))
+    hugeFuel
+
+theorem Mon.degree_k_eq_degree : Mon.degree_k m = Mon.degree m := by
+  unfold degree_k
+  generalize hugeFuel = fuel
+  induction fuel generalizing m with
+  | zero => rfl
+  | succ fuel ih =>
+    conv => rhs; unfold degree
+    split
+    · rfl
+    · dsimp only
+      rw [← ih]
+      rfl
+
 def Var.revlex (x y : Var) : Ordering :=
   bif x.blt y then .gt
   else bif y.blt x then .lt
@@ -270,7 +339,7 @@ noncomputable def Mon.grevlex_k (m₁ m₂ : Mon) : Ordering :=
   Bool.rec
     (Bool.rec .gt .lt (Nat.blt m₁.degree m₂.degree))
     (revlex_k m₁ m₂)
-    (Nat.beq m₁.degree m₂.degree)
+    (Nat.beq m₁.degree_k m₂.degree_k)
 
 theorem Mon.revlex_k_eq_revlex (m₁ m₂ : Mon) : m₁.revlex_k m₂ = m₁.revlex m₂ := by
   unfold revlex_k revlex
@@ -302,18 +371,18 @@ theorem Mon.grevlex_k_eq_grevlex (m₁ m₂ : Mon) : m₁.grevlex_k m₂ = m₁.
   next h =>
     have h₁ : Nat.blt m₁.degree m₂.degree = true := by simp [h]
     have h₂ : Nat.beq m₁.degree m₂.degree = false := by rw [← Bool.not_eq_true, Nat.beq_eq]; omega
-    simp [h₁, h₂]
+    simp [degree_k_eq_degree, h₁, h₂]
   next h =>
     split
     next h' =>
       have h₂ : Nat.beq m₁.degree m₂.degree = true := by rw [Nat.beq_eq, h']
-      simp [h₂]
+      simp [degree_k_eq_degree, h₂]
     next h' =>
       have h₁ : Nat.blt m₁.degree m₂.degree = false := by
         rw [← Bool.not_eq_true, Nat.blt_eq]; assumption
       have h₂ : Nat.beq m₁.degree m₂.degree = false := by
         rw [← Bool.not_eq_true, Nat.beq_eq]; assumption
-      simp [h₁, h₂]
+      simp [degree_k_eq_degree, h₁, h₂]
 
 inductive Poly where
   | num (k : Int)
@@ -481,7 +550,7 @@ noncomputable def Poly.mulMon_k (k : Int) (m : Mon) (p : Poly) : Poly :=
     (Bool.rec
       (Poly.rec
         (fun k' => Bool.rec (.add (Int.mul k k') m (.num 0)) (.num 0) (Int.beq' k' 0))
-        (fun k' m' _ ih => .add (Int.mul k k') (m.mul m') ih)
+        (fun k' m' _ ih => .add (Int.mul k k') (m.mul_k m') ih)
         p)
       (p.mulConst_k k)
       (Mon.beq' m .unit))
@@ -511,7 +580,7 @@ noncomputable def Poly.mulMon_k (k : Int) (m : Mon) (p : Poly) : Poly :=
         next =>
           have h : Int.beq' k 0 = false := by simp [*]
           simp [h]
-      next ih => simp [← ih]
+      next ih => simp [← ih, Mon.mul_k_eq_mul]
 
 def Poly.mulMon_nc (k : Int) (m : Mon) (p : Poly) : Poly :=
   bif k == 0 then
@@ -820,26 +889,22 @@ where
     | .num k' => acc.insert (k*k' % c) m
     | .add k' m' p => go p (acc.insert (k*k' % c) (m.mul_nc m'))
 
-def Poly.combineC (p₁ p₂ : Poly) (c : Nat) : Poly :=
-  go hugeFuel p₁ p₂
-where
-  go (fuel : Nat) (p₁ p₂ : Poly) : Poly :=
-    match fuel with
-    | 0 => p₁.concat p₂
-    | fuel + 1 => match p₁, p₂ with
-      | .num k₁, .num k₂ => .num ((k₁ + k₂) % c)
-      | .num k₁, .add k₂ m₂ p₂ => addConstC (.add k₂ m₂ p₂) k₁ c
-      | .add k₁ m₁ p₁, .num k₂ => addConstC (.add k₁ m₁ p₁) k₂ c
-      | .add k₁ m₁ p₁, .add k₂ m₂ p₂ =>
-        match m₁.grevlex m₂ with
-        | .eq =>
-          let k := (k₁ + k₂) % c
-          bif k == 0 then
-            go fuel p₁ p₂
-          else
-            .add k m₁ (go fuel p₁ p₂)
-        | .gt => .add k₁ m₁ (go fuel p₁ (.add k₂ m₂ p₂))
-        | .lt => .add k₂ m₂ (go fuel (.add k₁ m₁ p₁) p₂)
+@[semireducible]
+def Poly.combineC (p₁ p₂ : Poly) (c : Nat) : Poly := match p₁, p₂ with
+  | .num k₁, .num k₂ => .num ((k₁ + k₂) % c)
+  | .num k₁, .add k₂ m₂ p₂ => addConstC (.add k₂ m₂ p₂) k₁ c
+  | .add k₁ m₁ p₁, .num k₂ => addConstC (.add k₁ m₁ p₁) k₂ c
+  | .add k₁ m₁ p₁, .add k₂ m₂ p₂ =>
+    match m₁.grevlex m₂ with
+    | .eq =>
+      let k := (k₁ + k₂) % c
+      bif k == 0 then
+        combineC p₁ p₂ c
+      else
+        .add k m₁ (combineC p₁ p₂ c)
+    | .gt => .add k₁ m₁ (combineC p₁ (.add k₂ m₂ p₂) c)
+    | .lt => .add k₂ m₂ (combineC (.add k₁ m₁ p₁) p₂ c)
+termination_by sizeOf p₁ + sizeOf p₂
 
 def Poly.mulC (p₁ : Poly) (p₂ : Poly) (c : Nat) : Poly :=
   go p₁ (.num 0)
@@ -1363,10 +1428,9 @@ theorem Poly.denote_mulMonC_nc {α c} [Ring α] [IsCharP α c] (ctx : Context α
 
 theorem Poly.denote_combineC {α c} [Ring α] [IsCharP α c] (ctx : Context α) (p₁ p₂ : Poly)
     : (combineC p₁ p₂ c).denote ctx = p₁.denote ctx + p₂.denote ctx := by
-  unfold combineC; generalize hugeFuel = fuel
-  fun_induction combineC.go
-    <;> simp [*, denote_concat, denote_addConstC, denote, intCast_add,
-          add_comm, add_left_comm, add_assoc, IsCharP.intCast_emod, zsmul_eq_intCast_mul]
+  fun_induction combineC
+    <;> simp [*, denote_addConstC, denote, intCast_add, add_comm, add_left_comm, add_assoc,
+      IsCharP.intCast_emod, zsmul_eq_intCast_mul]
   next hg _ h _ =>
     simp +zetaDelta at h
     rw [← add_assoc, Mon.eq_of_grevlex hg, ← right_distrib, ← intCast_add,

src/Init/Grind/Util.lean

@@ -122,4 +122,15 @@ See comment at `alreadyNorm`
 theorem em (p : Prop) : alreadyNorm p ∨ alreadyNorm (¬ p) :=
   Classical.em p
 
+/--
+Marker for grind-solved subproofs in `exact? +grind` suggestions.
+When `exact?` uses grind as a discharger, it wraps the proof in this marker
+so that the unexpander can replace it with `(by grind)` in the suggestion.
+-/
+@[inline] def Marker {α : Sort u} (a : α) : α := a
+
+@[app_unexpander Marker]
+meta def markerUnexpander : PrettyPrinter.Unexpander := fun _ => do
+  `(by grind)
+
 end Lean.Grind

src/Init/Internal/Order/Basic.lean

@@ -64,10 +64,25 @@ This is intended to be used in the construction of `partial_fixpoint`, and not m
 -/
 def chain (c : α → Prop) : Prop := ∀ x y , c x → c y → x ⊑ y ∨ y ⊑ x
 
+def is_sup {α : Sort u} [PartialOrder α] (c : α → Prop) (s : α) : Prop :=
+  ∀ x, s ⊑ x ↔ (∀ y, c y → y ⊑ x)
+
+theorem is_sup_unique {α} [PartialOrder α] {c : α → Prop} {s₁ s₂ : α}
+    (h₁ : is_sup c s₁) (h₂ : is_sup c s₂) : s₁ = s₂ := by
+  apply PartialOrder.rel_antisymm
+  · apply (h₁ s₂).mpr
+    intro y hy
+    apply (h₂ s₂).mp PartialOrder.rel_refl y hy
+  · apply (h₂ s₁).mpr
+    intro y hy
+    apply (h₁ s₁).mp PartialOrder.rel_refl y hy
+
 end PartialOrder
 
 section CCPO
 
+open PartialOrder
+
 /--
 A chain-complete partial order (CCPO) is a partial order where every chain has a least upper bound.
 
@@ -76,67 +91,75 @@ otherwise.
 -/
 class CCPO (α : Sort u) extends PartialOrder α where
   /--
-  The least upper bound of a chain.
-
-  This is intended to be used in the construction of `partial_fixpoint`, and not meant to be used
-  otherwise.
+  The least upper bound of chains exists.
   -/
-  csup : (α → Prop) → α
-  /--
-  `csup c` is the least upper bound of the chain `c` when all elements `x` that are at
-  least as large as `csup c` are at least as large as all elements of `c`, and vice versa.
-  -/
-  csup_spec {c : α → Prop} (hc : chain c) : csup c ⊑ x ↔ (∀ y, c y → y ⊑ x)
+  has_csup {c : α → Prop} (hc : chain c) : Exists (is_sup c)
 
-open PartialOrder CCPO
+open  CCPO
 
 variable {α  : Sort u} [CCPO α]
 
-theorem csup_le {c : α → Prop} (hchain : chain c) : (∀ y, c y → y ⊑ x) → csup c ⊑ x :=
-  (csup_spec hchain).mpr
+noncomputable def CCPO.csup {c : α → Prop} (hc : chain c) : α :=
+  Classical.choose (CCPO.has_csup hc)
 
-theorem le_csup {c : α → Prop} (hchain : chain c) {y : α} (hy : c y) : y ⊑ csup c :=
-  (csup_spec hchain).mp rel_refl y hy
+theorem CCPO.csup_spec {c : α → Prop} (hc : chain c) : is_sup c (csup hc) :=
+  @fun x => Classical.choose_spec (CCPO.has_csup hc) x
+
+theorem csup_le {c : α → Prop} (hc : chain c) : (∀ y, c y → y ⊑ x) → csup hc ⊑ x :=
+  (csup_spec hc x).mpr
+
+theorem le_csup {c : α → Prop} (hc : chain c) {y : α} (hy : c y) : y ⊑ csup hc :=
+  (csup_spec hc (csup hc)).mp rel_refl y hy
+
+def empty_chain (α) : α → Prop := fun _ => False
+
+def chain_empty (α : Sort u) [PartialOrder α] : chain (empty_chain α) := by
+  intro x y hx hy; contradiction
 
 /--
 The bottom element is the least upper bound of the empty chain.
 
 This is intended to be used in the construction of `partial_fixpoint`, and not meant to be used otherwise.
 -/
-def bot : α := csup (fun _ => False)
+noncomputable def bot : α := csup (chain_empty α)
 
 scoped notation "⊥" => bot
 
 theorem bot_le (x : α) : ⊥ ⊑ x := by
   apply csup_le
-  · intro x y hx hy; contradiction
-  · intro x hx; contradiction
+  intro x y; contradiction
 
 end CCPO
 
 
 section CompleteLattice
+
 /--
 A complete lattice is a partial order where every subset has a least upper bound.
 -/
 class CompleteLattice (α : Sort u) extends PartialOrder α where
   /--
-  The least upper bound of an arbitrary subset in the complete_lattice.
+  The least upper bound of an arbitrary subset exists.
   -/
-  sup : (α → Prop) → α
-  sup_spec {c : α → Prop} : sup c ⊑ x ↔ (∀ y, c y → y ⊑ x)
+  has_sup (c : α → Prop) : Exists (is_sup c)
 
 open PartialOrder CompleteLattice
 
 variable {α  : Sort u} [CompleteLattice α]
 
-theorem sup_le {c : α → Prop} : (∀ y, c y → y ⊑ x) → sup c ⊑ x :=
-  (sup_spec).mpr
+noncomputable def CompleteLattice.sup (c : α → Prop) : α :=
+  Classical.choose (CompleteLattice.has_sup c)
 
-theorem le_sup {c : α → Prop} {y : α} (hy : c y) : y ⊑ sup c :=
-  sup_spec.mp rel_refl y hy
+theorem CompleteLattice.sup_spec (c : α → Prop) : is_sup c (sup c) :=
+  @fun x => Classical.choose_spec (CompleteLattice.has_sup c) x
 
-def inf (c : α → Prop) : α := sup (∀ y, c y → · ⊑ y)
+theorem sup_le (c : α → Prop) : (∀ y, c y → y ⊑ x) → sup c ⊑ x :=
+  Iff.mpr (sup_spec c x)
+
+theorem le_sup (c : α → Prop) {y : α} (hy : c y) : y ⊑ sup c :=
+  Iff.mp (sup_spec c (sup c)) rel_refl y hy
+
+noncomputable def inf (c : α → Prop) : α := sup (∀ y, c y → · ⊑ y)
 
 theorem inf_spec {c : α → Prop} : x ⊑ inf c ↔ (∀ y, c y → x ⊑ y) where
   mp := by
@@ -204,7 +227,7 @@ from this definition, and `P ⊥` is a separate condition of the induction predi
 This is intended to be used in the construction of `partial_fixpoint`, and not meant to be used otherwise.
 -/
 def admissible (P : α → Prop) :=
-  ∀ (c : α → Prop), chain c → (∀ x, c x → P x) → P (csup c)
+  ∀ (c : α → Prop) (hc : chain c), (∀ x, c x → P x) → P (csup hc)
 
 theorem admissible_const_true : admissible (fun (_ : α) => True) :=
   fun _ _ _ => trivial
@@ -220,7 +243,7 @@ theorem chain_conj (c P : α → Prop) (hchain : chain c) : chain (fun x => c x
   exact hchain x y hcx hcy
 
 theorem csup_conj (c P : α → Prop) (hchain : chain c) (h : ∀ x, c x → ∃ y, c y ∧ x ⊑ y ∧ P y) :
-    csup c = csup (fun x => c x ∧ P x) := by
+    csup hchain = csup (chain_conj c P hchain) := by
   apply rel_antisymm
   · apply csup_le hchain
     intro x hcx
@@ -281,12 +304,12 @@ variable {c : α → Prop}
 -- Note that monotonicity is not required for the definition of `lfp`
 -- but it is required to show that `lfp` is a fixpoint of `f`.
 
-def lfp (f : α → α) : α :=
+noncomputable def lfp (f : α → α) : α :=
   inf (fun c => f c ⊑ c)
 
 set_option linter.unusedVariables false in
 -- The following definition takes a witness that a function is monotone
-def lfp_monotone (f : α → α) (hm : monotone f) : α :=
+noncomputable def lfp_monotone (f : α → α) (hm : monotone f) : α :=
   lfp f
 
 -- Showing that `lfp` is a prefixed point makes use of monotonicity
@@ -355,7 +378,7 @@ This is intended to be used in the construction of `partial_fixpoint`, and not m
 -/
 inductive iterates (f : α → α) : α → Prop where
   | step : iterates f x → iterates f (f x)
-  | sup {c : α → Prop} (hc : chain c) (hi : ∀ x, c x → iterates f x) : iterates f (csup c)
+  | sup {c : α → Prop} (hc : chain c) (hi : ∀ x, c x → iterates f x) : iterates f (csup hc)
 
 theorem chain_iterates {f : α → α} (hf : monotone f) : chain (iterates f) := by
   intro x y hx hy
@@ -367,7 +390,7 @@ theorem chain_iterates {f : α → α} (hf : monotone f) : chain (iterates f) :=
       · left; apply hf; assumption
       · right; apply hf; assumption
     case sup c hchain hi ih2 =>
-      change f x ⊑ csup c ∨ csup c ⊑ f x
+      change f x ⊑ csup hchain ∨ csup hchain ⊑ f x
       by_cases h : ∃ z, c z ∧ f x ⊑ z
       · left
         obtain ⟨z, hz, hfz⟩ := h
@@ -384,7 +407,7 @@ theorem chain_iterates {f : α → α} (hf : monotone f) : chain (iterates f) :=
         next => contradiction
         next => assumption
   case sup c hchain hi ih =>
-    change rel (csup c) y ∨ rel y (csup c)
+    change rel (csup hchain) y ∨ rel y (csup hchain)
     by_cases h : ∃ z, c z ∧ rel y z
     · right
       obtain ⟨z, hz, hfz⟩ := h
@@ -423,7 +446,7 @@ definition is not very meaningful and it simplifies applying theorems like `fix_
 
 This is intended to be used in the construction of `partial_fixpoint`, and not meant to be used otherwise.
 -/
-def fix (f : α → α) (hmono : monotone f) := csup (iterates f)
+noncomputable def fix (f : α → α) (hmono : monotone f) := csup (chain_iterates hmono)
 
 /--
 The main fixpoint theorem for fixed points of monotone functions in chain-complete partial orders.
@@ -488,49 +511,66 @@ theorem chain_apply [∀ x, PartialOrder (β x)] {c : (∀ x, β x) → Prop} (h
   next h => left; apply h x
   next h => right; apply h x
 
-def fun_csup [∀ x, CCPO (β x)] (c : (∀ x, β x) → Prop) (x : α) :=
-  CCPO.csup (fun y => ∃ f, c f ∧ f x = y)
+noncomputable def fun_csup [∀ x, CCPO (β x)] (c : (∀ x, β x) → Prop) (hc : chain c) (x : α) :=
+  CCPO.csup (chain_apply hc x)
 
-def fun_sup [∀ x, CompleteLattice (β x)] (c : (∀ x, β x) → Prop) (x : α) :=
-  CompleteLattice.sup (fun y => ∃ f, c f ∧ f x = y)
+theorem fun_csup_is_sup [∀ x, CCPO (β x)] (c : (∀ x, β x) → Prop) (hc : chain c) :
+    is_sup c (fun_csup c hc) := by
+  intro f
+  constructor
+  next =>
+    intro hf g hg x
+    apply rel_trans _ (hf x); clear hf
+    apply le_csup (chain_apply hc x)
+    exact ⟨g, hg, rfl⟩
+  next =>
+    intro h x
+    apply csup_le (chain_apply hc x)
+    intro y ⟨z, hz, hyz⟩
+    subst y
+    apply h z hz
 
 instance instCCPOPi [∀ x, CCPO (β x)] : CCPO (∀ x, β x) where
-  csup := fun_csup
-  csup_spec := by
-    intro f c hc
-    constructor
-    next =>
-      intro hf g hg x
-      apply rel_trans _ (hf x); clear hf
-      apply le_csup (chain_apply hc x)
-      exact ⟨g, hg, rfl⟩
-    next =>
-      intro h x
-      apply csup_le (chain_apply hc x)
-      intro y ⟨z, hz, hyz⟩
-      subst y
-      apply h z hz
+  has_csup hc := ⟨fun_csup _ hc, fun_csup_is_sup _ hc⟩
+
+theorem fun_csup_eq [∀ x, CCPO (β x)] (c : (∀ x, β x) → Prop) (hc : chain c) :
+    fun_csup c hc = CCPO.csup hc := by
+  apply is_sup_unique (c := c)
+  · apply fun_csup_is_sup
+  · apply CCPO.csup_spec
+
+noncomputable def fun_sup [∀ x, CompleteLattice (β x)] (c : (∀ x, β x) → Prop) (x : α) :=
+  CompleteLattice.sup (fun y => ∃ f, c f ∧ f x = y)
+
+theorem fun_sup_is_sup [∀ x, CompleteLattice (β x)] (c : (∀ x, β x) → Prop) :
+    is_sup c (fun_sup c) := by
+  intro f
+  constructor
+  case mp =>
+    intro hf g hg x
+    apply rel_trans _ (hf x)
+    apply le_sup
+    exact ⟨g, hg, rfl⟩
+  case mpr =>
+    intro h x
+    apply sup_le
+    intro y ⟨z, hz, hyz⟩
+    subst y
+    apply h z hz
 
 instance instCompleteLatticePi [∀ x, CompleteLattice (β x)] : CompleteLattice (∀ x, β x) where
-  sup := fun_sup
-  sup_spec := by
-    intro f c
-    constructor
-    case mp =>
-      intro hf g hg x
-      apply rel_trans _ (hf x)
-      apply le_sup
-      exact ⟨g, hg, rfl⟩
-    case mpr =>
-      intro h x
-      apply sup_le
-      intro y ⟨z, hz, hyz⟩
-      subst y
-      apply h z hz
+  has_sup c := ⟨fun_sup c, fun_sup_is_sup c⟩
+
+theorem fun_sup_eq [∀ x, CompleteLattice (β x)] (c : (∀ x, β x) → Prop) :
+    fun_sup c = CompleteLattice.sup c := by
+  apply is_sup_unique (c := c)
+  · apply fun_sup_is_sup
+  · apply CompleteLattice.sup_spec
 
 def admissible_apply [∀ x, CCPO (β x)] (P : ∀ x, β x → Prop) (x : α)
   (hadm : admissible (P x)) : admissible (fun (f : ∀ x, β x) => P x (f x)) := by
   intro c hchain h
+  rw [← fun_csup_eq]
   apply hadm _ (chain_apply hchain x)
   rintro _ ⟨f, hcf, rfl⟩
   apply h _ hcf
@@ -622,67 +662,84 @@ theorem PProd.chain.chain_snd [CCPO α] [CCPO β] {c : α ×' β → Prop} (hcha
   case inl h => left; exact h.2
   case inr h => right; exact h.2
 
-instance instCompleteLatticePProd [CompleteLattice α] [CompleteLattice β] : CompleteLattice (α ×' β) where
-  sup c := ⟨CompleteLattice.sup (PProd.fst c), CompleteLattice.sup (PProd.snd c)⟩
-  sup_spec := by
-    intro ⟨a, b⟩ c
-    constructor
-    case mp =>
-      intro ⟨h₁, h₂⟩ ⟨a', b'⟩ cab
-      constructor <;> dsimp only at *
-      · apply rel_trans ?_ h₁
-        unfold PProd.fst at *
-        apply le_sup
-        apply Exists.intro b'
-        exact cab
-      . apply rel_trans ?_ h₂
-        apply le_sup
-        unfold PProd.snd at *
-        apply Exists.intro a'
-        exact cab
-    case mpr =>
-      intro h
-      constructor <;> dsimp only
-      . apply sup_le
-        unfold PProd.fst
-        intro y' ex
-        apply Exists.elim ex
-        intro b' hc
-        apply (h ⟨y', b' ⟩ hc).1
-      . apply sup_le
-        unfold PProd.snd
-        intro b' ex
-        apply Exists.elim ex
-        intro y' hc
-        apply (h ⟨y', b' ⟩ hc).2
+noncomputable def prod_csup [CCPO α] [CCPO β] (c : α ×' β → Prop) (hchain : chain c) : α ×' β :=
+  ⟨CCPO.csup (PProd.chain.chain_fst hchain), CCPO.csup (PProd.chain.chain_snd hchain)⟩
+
+theorem prod_csup_is_sup [CCPO α] [CCPO β] (c : α ×' β → Prop) (hchain : chain c) :
+    is_sup c (prod_csup c hchain) := by
+  intro ⟨a, b⟩
+  constructor
+  next =>
+    intro ⟨h₁, h₂⟩ ⟨a', b'⟩ cab
+    constructor <;> dsimp at *
+    · apply rel_trans ?_ h₁
+      apply le_csup (PProd.chain.chain_fst hchain)
+      exact ⟨b', cab⟩
+    · apply rel_trans ?_ h₂
+      apply le_csup (PProd.chain.chain_snd hchain)
+      exact ⟨a', cab⟩
+  next =>
+    intro h
+    constructor <;> dsimp
+    · apply csup_le (PProd.chain.chain_fst hchain)
+      intro a' ⟨b', hcab⟩
+      apply (h _ hcab).1
+    · apply csup_le (PProd.chain.chain_snd hchain)
+      intro b' ⟨a', hcab⟩
+      apply (h _ hcab).2
 
 instance instCCPOPProd [CCPO α] [CCPO β] : CCPO (α ×' β) where
-  csup c := ⟨CCPO.csup (PProd.chain.fst c), CCPO.csup (PProd.chain.snd c)⟩
-  csup_spec := by
-    intro ⟨a, b⟩ c hchain
-    constructor
-    next =>
-      intro ⟨h₁, h₂⟩ ⟨a', b'⟩ cab
-      constructor <;> dsimp at *
-      · apply rel_trans ?_ h₁
-        apply le_csup (PProd.chain.chain_fst hchain)
-        exact ⟨b', cab⟩
-      · apply rel_trans ?_ h₂
-        apply le_csup (PProd.chain.chain_snd hchain)
-        exact ⟨a', cab⟩
-    next =>
-      intro h
-      constructor <;> dsimp
-      · apply csup_le (PProd.chain.chain_fst hchain)
-        intro a' ⟨b', hcab⟩
-        apply (h _ hcab).1
-      · apply csup_le (PProd.chain.chain_snd hchain)
-        intro b' ⟨a', hcab⟩
-        apply (h _ hcab).2
+  has_csup hchain := ⟨prod_csup _ hchain, prod_csup_is_sup _ hchain⟩
+
+theorem prod_csup_eq [CCPO α] [CCPO β] (c : α ×' β → Prop) (hchain : chain c) :
+    prod_csup c hchain = CCPO.csup hchain := by
+  apply is_sup_unique (c := c)
+  · apply prod_csup_is_sup
+  · apply CCPO.csup_spec
+
+noncomputable def prod_sup [CompleteLattice α] [CompleteLattice β] (c : α ×' β → Prop) : α ×' β :=
+  ⟨CompleteLattice.sup (PProd.fst c), CompleteLattice.sup (PProd.snd c)⟩
+
+theorem prod_sup_is_sup [CompleteLattice α] [CompleteLattice β] (c : α ×' β → Prop) :
+    is_sup c (prod_sup c) := by
+  intro ⟨a, b⟩
+  constructor
+  case mp =>
+    intro ⟨h₁, h₂⟩ ⟨a', b'⟩ cab
+    constructor <;> dsimp only at *
+    · apply rel_trans ?_ h₁
+      unfold prod_sup PProd.fst at *
+      apply le_sup
+      apply Exists.intro b'
+      exact cab
+    . apply rel_trans ?_ h₂
+      apply le_sup
+      unfold PProd.snd at *
+      apply Exists.intro a'
+      exact cab
+  case mpr =>
+    intro h
+    constructor <;> dsimp only
+    . apply sup_le
+      unfold PProd.fst
+      intro y' ex
+      apply Exists.elim ex
+      intro b' hc
+      apply (h ⟨y', b' ⟩ hc).1
+    . apply sup_le
+      unfold PProd.snd
+      intro b' ex
+      apply Exists.elim ex
+      intro y' hc
+      apply (h ⟨y', b' ⟩ hc).2
+
+instance instCompleteLatticePProd [CompleteLattice α] [CompleteLattice β] : CompleteLattice (α ×' β) where
+  has_sup c := ⟨prod_sup c, prod_sup_is_sup c⟩
 
 theorem admissible_pprod_fst {α : Sort u} {β : Sort v} [CCPO α] [CCPO β] (P : α → Prop)
     (hadm : admissible P) : admissible (fun (x : α ×' β) => P x.1) := by
   intro c hchain h
+  rw [<- prod_csup_eq]
   apply hadm _ (PProd.chain.chain_fst hchain)
   intro x ⟨y, hxy⟩
   apply h ⟨x,y⟩ hxy
@@ -690,6 +747,7 @@ theorem admissible_pprod_fst {α : Sort u} {β : Sort v} [CCPO α] [CCPO β] (P
 theorem admissible_pprod_snd {α : Sort u} {β : Sort v} [CCPO α] [CCPO β] (P : β → Prop)
     (hadm : admissible P) : admissible (fun (x : α ×' β) => P x.2) := by
   intro c hchain h
+  rw [<- prod_csup_eq]
   apply hadm _ (PProd.chain.chain_snd hchain)
   intro y ⟨x, hxy⟩
   apply h ⟨x,y⟩ hxy
@@ -736,49 +794,57 @@ noncomputable def flat_csup (c : FlatOrder b → Prop) : FlatOrder b := by
   · exact Classical.choose h
   · exact b
 
-noncomputable instance FlatOrder.instCCPO : CCPO (FlatOrder b) where
-  csup := flat_csup
-  csup_spec := by
-    intro x c hc
-    unfold flat_csup
-    split
-    next hex =>
-      apply Classical.some_spec₂ (q := (· ⊑ x ↔ (∀ y, c y → y ⊑ x)))
-      clear hex
-      intro z ⟨hz, hnb⟩
-      constructor
-      · intro h y hy
-        apply PartialOrder.rel_trans _ h; clear h
-        cases hc y z hy hz
-        next => assumption
-        next h =>
-          cases h
-          · contradiction
-          · constructor
-      · intro h
-        cases h z hz
+theorem flat_csup_is_sup (c : FlatOrder b → Prop) (hc : chain c) :
+    is_sup c (flat_csup c) := by
+  intro x
+  unfold flat_csup
+  split
+  next hex =>
+    apply Classical.some_spec₂ (q := (· ⊑ x ↔ (∀ y, c y → y ⊑ x)))
+    clear hex
+    intro z ⟨hz, hnb⟩
+    constructor
+    · intro h y hy
+      apply PartialOrder.rel_trans _ h; clear h
+      cases hc y z hy hz
+      next => assumption
+      next h =>
+        cases h
         · contradiction
         · constructor
-    next hnotex =>
-      constructor
-      · intro h y hy; clear h
-        suffices y = b by rw [this]; exact rel.bot
-        rw [not_exists] at hnotex
-        specialize hnotex y
-        rw [not_and] at hnotex
-        specialize hnotex hy
-        rw [@Classical.not_not] at hnotex
-        assumption
-      · intro; exact rel.bot
+    · intro h
+      cases h z hz
+      · contradiction
+      · constructor
+  next hnotex =>
+    constructor
+    · intro h y hy; clear h
+      suffices y = b by rw [this]; exact FlatOrder.rel.bot
+      rw [not_exists] at hnotex
+      specialize hnotex y
+      rw [not_and] at hnotex
+      specialize hnotex hy
+      rw [@Classical.not_not] at hnotex
+      assumption
+    · intro; exact FlatOrder.rel.bot
+
+instance FlatOrder.instCCPO : CCPO (FlatOrder b) where
+  has_csup hchain := ⟨flat_csup _ , flat_csup_is_sup _ hchain⟩
+
+theorem flat_csup_eq (c : FlatOrder b → Prop) (hchain : chain c) :
+    flat_csup c = CCPO.csup hchain := by
+  apply is_sup_unique (c := c)
+  · apply flat_csup_is_sup _ hchain
+  · apply CCPO.csup_spec
 
 theorem admissible_flatOrder (P : FlatOrder b → Prop) (hnot : P b) : admissible P := by
   intro c hchain h
   by_cases h' : ∃ (x : FlatOrder b), c x ∧ x ≠ b
-  · simp [CCPO.csup, flat_csup, h']
+  · simp [← flat_csup_eq, flat_csup, h']
     apply Classical.some_spec₂ (q := (P ·))
     intro x ⟨hcx, hneb⟩
     apply h x hcx
-  · simp [CCPO.csup, flat_csup, h', hnot]
+  · simp [← flat_csup_eq, flat_csup, h', hnot]
 
 end flat_order
 
@@ -809,8 +875,8 @@ theorem monotone_bind
   · apply MonoBind.bind_mono_right (fun y => monotone_apply y _ hmono₂ _ _ hx₁₂)
 
 instance : PartialOrder (Option α) := inferInstanceAs (PartialOrder (FlatOrder none))
-noncomputable instance : CCPO (Option α) := inferInstanceAs (CCPO (FlatOrder none))
-noncomputable instance : MonoBind Option where
+instance : CCPO (Option α) := inferInstanceAs (CCPO (FlatOrder none))
+instance : MonoBind Option where
   bind_mono_left h := by
     cases h
     · exact FlatOrder.rel.bot
@@ -921,12 +987,22 @@ theorem monotone_stateTRun [PartialOrder γ]
     monotone (fun (x : γ) => StateT.run (f x) s) :=
   monotone_apply s _ hmono
 
--- TODO: axiomatize these instances (ideally without `Nonempty ε`) when EIO and friends are opaque
+noncomputable def EST.bot [Nonempty ε] : EST ε σ α :=
+  fun s => .error Classical.ofNonempty (Classical.choice ⟨s⟩)
 
-noncomputable instance [Nonempty ε] : CCPO (EST ε σ α) :=
-  inferInstanceAs (CCPO ((s : _) → FlatOrder (.error Classical.ofNonempty (Classical.choice ⟨s⟩))))
+-- Essentially
+--   instance [Nonempty ε] : CCPO (EST ε σ α) :=
+--     inferInstanceAs (CCPO ((s : _) → FlatOrder (EST.bot s)))
+-- but hat would incur a noncomputable on the instance
 
-noncomputable instance [Nonempty ε] : MonoBind (EST ε σ) where
+instance [Nonempty ε] : CCPO (EST ε σ α) where
+  rel := PartialOrder.rel (α := ∀ s, FlatOrder (EST.bot s))
+  rel_refl := PartialOrder.rel_refl
+  rel_antisymm := PartialOrder.rel_antisymm
+  rel_trans := PartialOrder.rel_trans
+  has_csup hchain := CCPO.has_csup (α := ∀ s, FlatOrder (EST.bot s)) hchain
+
+instance [Nonempty ε] : MonoBind (EST ε σ) where
   bind_mono_left {_ _ a₁ a₂ f} h₁₂ := by
     intro s
     specialize h₁₂ s
@@ -944,18 +1020,18 @@ noncomputable instance [Nonempty ε] : MonoBind (EST ε σ) where
     · apply h₁₂
     · exact .refl
 
-noncomputable instance [Nonempty α] : CCPO (ST σ α) :=
-  inferInstanceAs (CCPO ((s : _) → FlatOrder (.mk Classical.ofNonempty (Classical.choice ⟨s⟩))))
-
-noncomputable instance [Nonempty α] : CCPO (BaseIO α) :=
-  inferInstanceAs (CCPO (ST IO.RealWorld α))
-
-noncomputable instance [Nonempty ε] : CCPO (EIO ε α) :=
+instance [Nonempty ε] : CCPO (EIO ε α) :=
   inferInstanceAs (CCPO (EST ε IO.RealWorld α))
 
-noncomputable instance [Nonempty ε] : MonoBind (EIO ε) :=
+instance [Nonempty ε] : MonoBind (EIO ε) :=
   inferInstanceAs (MonoBind (EST ε IO.RealWorld))
 
+instance : CCPO (IO α) :=
+  inferInstanceAs (CCPO (EIO IO.Error α))
+
+instance : MonoBind IO :=
+  inferInstanceAs (MonoBind (EIO IO.Error))
+
 end mono_bind
 
 section implication_order
@@ -970,9 +1046,9 @@ instance ImplicationOrder.instOrder : PartialOrder ImplicationOrder where
 
 -- This defines a complete lattice on `Prop`, used to define inductive predicates
 instance ImplicationOrder.instCompleteLattice : CompleteLattice ImplicationOrder where
-  sup c := ∃ p, c p ∧ p
-  sup_spec := by
-    intro x c
+  has_sup c := by
+    exists ∃ p, c p ∧ p
+    intro x
     constructor
     case mp =>
       intro h y cy hy
@@ -1032,9 +1108,9 @@ instance ReverseImplicationOrder.instOrder : PartialOrder ReverseImplicationOrde
 
 -- This defines a complete lattice on `Prop`, used to define coinductive predicates
 instance ReverseImplicationOrder.instCompleteLattice : CompleteLattice ReverseImplicationOrder where
-  sup c := ∀ p, c p → p
-  sup_spec := by
-    intro x c
+  has_sup c := by
+    exists ∀ p, c p → p
+    intro x
     constructor
     case mp =>
       intro h y cy l

src/Init/Notation.lean

@@ -633,6 +633,20 @@ existing code. It may be removed in a future version of the library.
 syntax (name := deprecated) "deprecated" (ppSpace ident)? (ppSpace str)?
     (" (" &"since" " := " str ")")? : attr
 
+/--
+The attribute `@[suggest_for]` on a declaration suggests likely ways in which
+someone might **incorrectly** refer to a definition.
+
+* `@[suggest_for String.endPos]` on the definition of `String.rawEndPos` suggests that `"str".endPos` might be correctable to `"str".rawEndPos`.
+* `@[suggest_for Either Result]` on the definition of `Except` suggests that `Either Nat String` might be correctable to `Except Nat String`.
+
+The namespace of the suggestions is always relative to the root namespace. In the namespace `X.Y`,
+adding an annotation `@[suggest_for Z.bar]` to `def Z.foo` will suggest `X.Y.Z.foo` only as a
+replacement for `Z.foo`. If your intent is to suggest `X.Y.Z.foo` as a replacement for
+`X.Y.Z.bar`, you must instead use the annotation `@[suggest_for X.Y.Z.bar]`.
+-/
+syntax (name := suggest_for) "suggest_for" (ppSpace ident)+ : attr
+
 /--
 The `@[coe]` attribute on a function (which should also appear in a
 `instance : Coe A B := ⟨myFn⟩` declaration) allows the delaborator to show
@@ -842,7 +856,7 @@ Position reporting:
   `#guard_msgs` appears.
 - `positions := false` does not report position info.
 
-For example, `#guard_msgs (error, drop all) in cmd` means to check warnings and drop
+For example, `#guard_msgs (error, drop all) in cmd` means to check errors and drop
 everything else.
 
 The command elaborator has special support for `#guard_msgs` for linting.

src/Init/Prelude.lean

@@ -1837,6 +1837,8 @@ def Nat.ble : @& Nat → @& Nat → Bool
   | succ _, zero   => false
   | succ n, succ m => ble n m
 
+attribute [gen_constructor_elims] Bool
+
 /--
 Non-strict, or weak, inequality of natural numbers, usually accessed via the `≤` operator.
 -/
@@ -1860,9 +1862,14 @@ protected def Nat.lt (n m : Nat) : Prop :=
 instance instLTNat : LT Nat where
   lt := Nat.lt
 
-theorem Nat.not_succ_le_zero : ∀ (n : Nat), LE.le (succ n) 0 → False
-  | 0      => nofun
-  | succ _ => nofun
+theorem Nat.not_succ_le_zero (n : Nat) : LE.le (succ n) 0 → False :=
+  -- No injectivity tactic until `attribute [gen_constructor_elims] Nat`
+  have : ∀ m, Eq m 0 → LE.le (succ n) m → False := fun _ hm hle =>
+    Nat.le.casesOn (motive := fun m _ => Eq m 0 → False) hle
+      (fun h => Nat.noConfusion h)
+      (fun _ h => Nat.noConfusion h)
+      hm
+  this 0 rfl
 
 theorem Nat.not_lt_zero (n : Nat) : Not (LT.lt n 0) :=
   not_succ_le_zero n
@@ -1999,10 +2006,12 @@ protected theorem Nat.lt_of_not_le {a b : Nat} (h : Not (LE.le a b)) : LT.lt b a
 
 protected theorem Nat.add_pos_right :
     {b : Nat} → (a : Nat) → (hb : LT.lt 0 b) → LT.lt 0 (HAdd.hAdd a b)
+  | zero, _, h => (Nat.not_succ_le_zero _ h).elim
   | succ _, _, _ => Nat.zero_lt_succ _
 
 protected theorem Nat.mul_pos :
     {n m : Nat} → (hn : LT.lt 0 n) → (hm : LT.lt 0 m) → LT.lt 0 (HMul.hMul n m)
+  | _, zero, _, hb => (Nat.not_succ_le_zero _ hb).elim
   | _, succ _, ha, _ => Nat.add_pos_right _ ha
 
 protected theorem Nat.pow_pos {a : Nat} : {n : Nat} → (h : LT.lt 0 a) → LT.lt 0 (HPow.hPow a n)
@@ -2059,6 +2068,8 @@ protected def Nat.sub : (@& Nat) → (@& Nat) → Nat
   | a, 0      => a
   | a, succ b => pred (Nat.sub a b)
 
+attribute [gen_constructor_elims] Nat
+
 instance instSubNat : Sub Nat where
   sub := Nat.sub
 
@@ -2216,9 +2227,6 @@ theorem Nat.mod_lt : (x : Nat) →  {y : Nat} → (hy : LT.lt 0 y) → LT.lt (HM
       | .isTrue _ => Nat.modCore_lt hm
       | .isFalse h => Nat.lt_of_not_le h
 
-attribute [gen_constructor_elims] Nat
-attribute [gen_constructor_elims] Bool
-
 /--
 Gets the word size of the current platform. The word size may be 64 or 32 bits.
 
@@ -3177,7 +3185,7 @@ This is a cached value, so it is `O(1)` to access. The space allocated for an ar
 its _capacity_, is at least as large as its size, but may be larger. The capacity of an array is an
 internal detail that's not observable by Lean code.
 -/
-@[extern "lean_array_get_size"]
+@[extern "lean_array_get_size", tagged_return]
 def Array.size {α : Type u} (a : @& Array α) : Nat :=
  a.toList.length
 
@@ -3385,7 +3393,7 @@ Returns the number of bytes in the byte array.
 This is the number of bytes actually in the array, as distinct from its capacity, which is the
 amount of memory presently allocated for the array.
 -/
-@[extern "lean_byte_array_size"]
+@[extern "lean_byte_array_size", tagged_return]
 def ByteArray.size : (@& ByteArray) → Nat
   | ⟨bs⟩ => bs.size
 
@@ -3532,7 +3540,7 @@ The number of bytes used by the string's UTF-8 encoding.
 
 At runtime, this function takes constant time because the byte length of strings is cached.
 -/
-@[extern "lean_string_utf8_byte_size"]
+@[extern "lean_string_utf8_byte_size", tagged_return]
 def String.utf8ByteSize (s : @& String) : Nat :=
   s.toByteArray.size
 

src/Init/System/IO.lean

@@ -837,7 +837,7 @@ Encountering an EOF does not close a handle. Subsequent reads may block and retu
 -/
 @[extern "lean_io_prim_handle_read"] opaque read (h : @& Handle) (bytes : USize) : IO ByteArray
 /--
-Writes the provided bytes to the the handle.
+Writes the provided bytes to the handle.
 
 Writing to a handle is typically buffered, and may not immediately modify the file on disk. Use
 `IO.FS.Handle.flush` to write changes to buffers to the associated device.

src/Init/Tactics.lean

@@ -1004,7 +1004,7 @@ You can use `with` to provide the variables names for each constructor.
 syntax (name := cases) "cases " elimTarget,+ (" using " term)? (inductionAlts)? : tactic
 
 /--
-The `fun_induction` tactic is a convenience wrapper around the `induction` tactic to use the the
+The `fun_induction` tactic is a convenience wrapper around the `induction` tactic to use the
 functional induction principle.
 
 The tactic invocation
@@ -1690,6 +1690,21 @@ a lemma from the list until it gets stuck.
 syntax (name := applyRules) "apply_rules" optConfig (&" only")? (args)? (using_)? : tactic
 end SolveByElim
 
+/--
+Configuration for the `exact?` and `apply?` tactics.
+-/
+structure LibrarySearchConfig where
+  /-- If true, use `grind` as a discharger for subgoals that cannot be closed
+  by `solve_by_elim` alone. -/
+  grind : Bool := false
+  /-- If true, use `try?` as a discharger for subgoals that cannot be closed
+  by `solve_by_elim` alone. -/
+  try? : Bool := false
+  /-- If true (the default), star-indexed lemmas (with overly-general discrimination keys
+  like `[*]`) are searched as a fallback when no concrete-keyed lemmas are found.
+  Use `-star` to disable this fallback. -/
+  star : Bool := true
+
 /--
 Searches environment for definitions or theorems that can solve the goal using `exact`
 with conditions resolved by `solve_by_elim`.
@@ -1697,8 +1712,12 @@ with conditions resolved by `solve_by_elim`.
 The optional `using` clause provides identifiers in the local context that must be
 used by `exact?` when closing the goal.  This is most useful if there are multiple
 ways to resolve the goal, and one wants to guide which lemma is used.
+
+Use `+grind` to enable `grind` as a fallback discharger for subgoals.
+Use `+try?` to enable `try?` as a fallback discharger for subgoals.
+Use `-star` to disable fallback to star-indexed lemmas (like `Empty.elim`, `And.left`).
 -/
-syntax (name := exact?) "exact?" (" using " (colGt ident),+)? : tactic
+syntax (name := exact?) "exact?" optConfig (" using " (colGt ident),+)? : tactic
 
 /--
 Searches environment for definitions or theorems that can refine the goal using `apply`
@@ -1706,8 +1725,12 @@ with conditions resolved when possible with `solve_by_elim`.
 
 The optional `using` clause provides identifiers in the local context that must be
 used when closing the goal.
+
+Use `+grind` to enable `grind` as a fallback discharger for subgoals.
+Use `+try?` to enable `try?` as a fallback discharger for subgoals.
+Use `-star` to disable fallback to star-indexed lemmas.
 -/
-syntax (name := apply?) "apply?" (" using " (colGt term),+)? : tactic
+syntax (name := apply?) "apply?" optConfig (" using " (colGt term),+)? : tactic
 
 /--
 Syntax for excluding some names, e.g. `[-my_lemma, -my_theorem]`.

src/Init/Try.lean

@@ -82,3 +82,18 @@ macro "∎" : tactic => `(tactic| try?)
 /-- `∎` (typed as `\qed`) is a macro that expands to `by try? (wrapWithBy := true)` in term mode.
     The `wrapWithBy` config option causes suggestions to be prefixed with `by`. -/
 macro "∎" : term => `(by try? (wrapWithBy := true))
+
+namespace Lean.Try
+
+/--
+Marker for try?-solved subproofs in `exact? +try?` suggestions.
+When `exact?` uses try? as a discharger, it wraps the proof in this marker
+so that the unexpander can replace it with `(by try?)` in the suggestion.
+-/
+@[inline] def Marker {α : Sort u} (a : α) : α := a
+
+@[app_unexpander Marker]
+meta def markerUnexpander : PrettyPrinter.Unexpander := fun _ => do
+  `(by try?)
+
+end Lean.Try

src/Init/WF.lean

@@ -439,6 +439,53 @@ end
 
 end PSigma
 
+namespace WellFounded
+
+variable {α : Sort u}
+variable {motive : α → Sort v}
+variable (h : α → Nat)
+variable (F : (x : α) → ((y : α) → InvImage (· < ·) h y x → motive y) → motive x)
+
+/-- Helper gadget that prevents reduction of `Nat.eager n` unless `n` evalutes to a ground term. -/
+def Nat.eager (n : Nat) : Nat :=
+  if Nat.beq n n = true then n else n
+
+theorem Nat.eager_eq (n : Nat) : Nat.eager n = n := ite_self n
+
+/--
+A well-founded fixpoint operator specialized for `Nat`-valued measures. Given a measure `h`, it expects
+its higher order function argument `F` to invoke its argument only on values `y` that are smaller
+than `x` with regard to `h`.
+
+In contrast to to `WellFounded.fix`, this fixpoint operator reduces on closed terms. (More precisely:
+when `h x` evalutes to a ground value)
+
+-/
+def Nat.fix : (x : α) → motive x :=
+  let rec go : ∀ (fuel : Nat) (x : α), (h x < fuel) → motive x :=
+    Nat.rec
+      (fun _ hfuel => (Nat.not_succ_le_zero _ hfuel).elim)
+      (fun _ ih x hfuel => F x (fun y hy => ih y (Nat.lt_of_lt_of_le hy (Nat.le_of_lt_add_one hfuel))))
+  fun x => go (Nat.eager (h x + 1)) x (Nat.eager_eq _ ▸ Nat.lt_add_one _)
+
+protected theorem Nat.fix.go_congr (x : α) (fuel₁ fuel₂ : Nat) (h₁ : h x < fuel₁) (h₂ : h x < fuel₂) :
+    Nat.fix.go h F fuel₁ x h₁ = Nat.fix.go h F fuel₂ x h₂ := by
+  induction fuel₁ generalizing x fuel₂ with
+  | zero => contradiction
+  | succ fuel₁ ih =>
+    cases fuel₂ with
+    | zero => contradiction
+    | succ fuel₂ =>
+      exact congrArg (F x) (funext fun y => funext fun hy => ih y fuel₂ _ _ )
+
+theorem Nat.fix_eq (x : α) :
+    Nat.fix h F x = F x (fun y _ => Nat.fix h F y) := by
+  unfold Nat.fix
+  simp [Nat.eager_eq]
+  exact congrArg (F x) (funext fun _ => funext fun _ => Nat.fix.go_congr ..)
+
+end WellFounded
+
 /--
 The `wfParam` gadget is used internally during the construction of recursive functions by
 wellfounded recursion, to keep track of the parameter for which the automatic introduction

src/Init/WFExtrinsicFix.lean

@@ -0,0 +1,245 @@
+/-
+Copyright (c) 2025 Lean FRO, LLC. All rights reserved.
+Released under Apache 2.0 license as described in the file LICENSE.
+Authors: Paul Reichert
+-/
+module
+
+prelude
+public import Init.WF
+import Init.Classical
+import Init.Ext
+import Init.NotationExtra
+
+set_option doc.verso true
+set_option linter.missingDocs true
+
+/-!
+This module provides a fixpoint combinator that combines advantages of `partial` and well-founded
+recursion.
+
+The combinator is similar to {lean}`WellFounded.fix`, but does
+not require a well-foundedness proof. Therefore, it can be used in situations in which
+a proof of termination is impossible or inconvenient. Given a well-foundedness proof, there is a
+theorem guaranteeing that it is equal to {lean}`WellFounded.fix`.
+-/
+
+variable {α : Sort _} {β : α → Sort _} {γ : (a : α) → β a → Sort _}
+  {C : α → Sort _} {C₂ : (a : α) → β a → Sort _} {C₃ : (a : α) → (b : β a) → γ a b → Sort _}
+
+set_option doc.verso true
+
+namespace WellFounded
+
+/--
+The function implemented as the loop {lean}`opaqueFix R F a = F a (fun a _ => opaqueFix R F a)`.
+{lean}`opaqueFix R F` is the fixpoint of the functional {name}`F`, as long as the loop is
+well-founded.
+
+The loop might run forever depending on {name}`F`. It is opaque, i.e., it is impossible to prove
+nontrivial properties about it.
+-/
+@[specialize]
+partial def opaqueFix [∀ a, Nonempty (C a)] (R : α → α → Prop)
+    (F : ∀ a, (∀ a', R a' a → C a') → C a) (a : α) : C a :=
+  F a (fun a _ => opaqueFix R F a)
+
+/-
+SAFE assuming that the code generated by iteration over `F` is equivalent to the well-founded
+fixpoint of `F` if `R` is well-founded.
+-/
+/--
+A fixpoint combinator that can be used to construct recursive functions with an *extrinsic* proof
+of termination.
+
+Given a relation {name}`R` and a fixpoint functional {name}`F` which must be decreasing with respect
+to {name}`R`, {lean}`extrinsicFix R F` is the recursive function obtained by having {name}`F` call
+itself recursively.
+
+If {name}`R` is not well-founded, {lean}`extrinsicFix R F` might run forever. In this case,
+nothing interesting can be proved about the recursive function; it is opaque.
+
+If {name}`R` _is_ well-founded, {lean}`extrinsicFix R F` is equivalent to
+{lean}`WellFounded.fix _ F`, logically and regarding its termination behavior.
+-/
+@[implemented_by opaqueFix]
+public def extrinsicFix [∀ a, Nonempty (C a)] (R : α → α → Prop)
+    (F : ∀ a, (∀ a', R a' a → C a') → C a) (a : α) : C a :=
+  open scoped Classical in
+  if h : WellFounded R then
+    h.fix F a
+  else
+    -- Return `opaqueFix R F a` so that `implemented_by opaqueFix` is sound.
+    -- In effect, `extrinsicFix` is opaque if `WellFounded R` is false.
+    opaqueFix R F a
+
+public theorem extrinsicFix_eq_fix [∀ a, Nonempty (C a)] {R : α → α → Prop}
+    {F : ∀ a, (∀ a', R a' a → C a') → C a}
+    (wf : WellFounded R) {a : α} :
+    extrinsicFix R F a = wf.fix F a := by
+  simp only [extrinsicFix, dif_pos wf]
+
+public theorem extrinsicFix_eq_apply [∀ a, Nonempty (C a)] {R : α → α → Prop}
+    {F : ∀ a, (∀ a', R a' a → C a') → C a} (h : WellFounded R) {a : α} :
+    extrinsicFix R F a = F a (fun a _ => extrinsicFix R F a) := by
+  simp only [extrinsicFix, dif_pos h]
+  rw [WellFounded.fix_eq]
+
+/--
+A fixpoint combinator that allows for deferred proofs of termination.
+
+{lean}`extrinsicFix R F` is function implemented as the loop
+{lean}`extrinsicFix R F a = F a (fun a _ => extrinsicFix R F a)`.
+
+If the loop can be shown to be well-founded, {name}`extrinsicFix_eq_apply` proves that it satisfies the
+fixpoint equation. Otherwise, {lean}`extrinsicFix R F` is opaque, i.e., it is impossible to prove
+nontrivial properties about it.
+-/
+add_decl_doc extrinsicFix
+
+/--
+The function implemented as the loop
+{lean}`opaqueFix₂ R F a b = F a b (fun a' b' _ => opaqueFix₂ R F a' b')`.
+{lean}`opaqueFix₂ R F` is the fixpoint of the functional {name}`F`, as long as the loop is
+well-founded.
+
+The loop might run forever depending on {name}`F`. It is opaque, i.e., it is impossible to prove
+nontrivial properties about it.
+-/
+@[specialize]
+partial def opaqueFix₂ [∀ a b, Nonempty (C₂ a b)]
+    (R : (a : α) ×' β a → (a : α) ×' β a → Prop)
+    (F : (a : α) → (b : β a) → ((a' : α) → (b' : β a') → R ⟨a', b'⟩ ⟨a, b⟩ → C₂ a' b') → C₂ a b)
+    (a : α) (b : β a) :
+    C₂ a b :=
+  F a b (fun a' b' _ => opaqueFix₂ R F a' b')
+
+/-
+SAFE assuming that the code generated by iteration over `F` is equivalent to the well-founded
+fixpoint of `F` if `R` is well-founded.
+-/
+/--
+A fixpoint combinator that can be used to construct recursive functions of arity two with an
+*extrinsic* proof of termination.
+
+Given a relation {name}`R` and a fixpoint functional {name}`F` which must be decreasing with respect
+to {name}`R`, {lean}`extrinsicFix₂ R F` is the recursive function obtained by having {name}`F` call
+itself recursively.
+
+If {name}`R` is not well-founded, {lean}`extrinsicFix₂ R F` might run forever. In this case,
+nothing interesting can be proved about the recursive function; it is opaque.
+
+If {name}`R` _is_ well-founded, {lean}`extrinsicFix₂ R F` is equivalent to a well-founded recursive
+function, logically and regarding its termination behavior.
+-/
+@[implemented_by opaqueFix₂]
+public def extrinsicFix₂ [∀ a b, Nonempty (C₂ a b)]
+    (R : (a : α) ×' β a → (a : α) ×' β a → Prop)
+    (F : (a : α) → (b : β a) → ((a' : α) → (b' : β a') → R ⟨a', b'⟩ ⟨a, b⟩ → C₂ a' b') → C₂ a b)
+    (a : α) (b : β a) :
+    C₂ a b :=
+  let F' (x : PSigma β) (G : (y : PSigma β) → R y x → C₂ y.1 y.2) : C₂ x.1 x.2 :=
+    F x.1 x.2 (fun a b => G ⟨a, b⟩)
+  extrinsicFix (C := fun x : PSigma β => C₂ x.1 x.2) R F' ⟨a, b⟩
+
+public theorem extrinsicFix₂_eq_fix [∀ a b, Nonempty (C₂ a b)]
+    {R : (a : α) ×' β a → (a : α) ×' β a → Prop}
+    {F : ∀ a b, (∀ a' b', R ⟨a', b'⟩ ⟨a, b⟩ → C₂ a' b') → C₂ a b}
+    (wf : WellFounded R) {a b} :
+    extrinsicFix₂ R F a b = wf.fix (fun x G => F x.1 x.2 (fun a b h => G ⟨a, b⟩ h)) ⟨a, b⟩ := by
+  rw [extrinsicFix₂, extrinsicFix_eq_fix wf]
+
+public theorem extrinsicFix₂_eq_apply [∀ a b, Nonempty (C₂ a b)]
+    {R : (a : α) ×' β a → (a : α) ×' β a → Prop}
+    {F : (a : α) → (b : β a) → ((a' : α) → (b' : β a') → R ⟨a', b'⟩ ⟨a, b⟩ → C₂ a' b') → C₂ a b}
+    (wf : WellFounded R) {a : α} {b : β a} :
+    extrinsicFix₂ R F a b = F a b (fun a' b' _ => extrinsicFix₂ R F a' b') := by
+  rw [extrinsicFix₂, extrinsicFix_eq_apply wf]
+  rfl
+
+/--
+A fixpoint combinator that allows for deferred proofs of termination.
+
+{lean}`extrinsicFix₂ R F` is function implemented as the loop
+{lean}`extrinsicFix₂ R F a b = F a b (fun a' b' _ => extrinsicFix₂ R F a' b')`.
+
+If the loop can be shown to be well-founded, {name}`extrinsicFix₂_eq_apply` proves that it satisfies the
+fixpoint equation. Otherwise, {lean}`extrinsicFix₂ R F` is opaque, i.e., it is impossible to prove
+nontrivial properties about it.
+-/
+add_decl_doc extrinsicFix₂
+
+/--
+The function implemented as the loop
+{lean}`opaqueFix₃ R F a b c = F a b c (fun a' b' c' _ => opaqueFix₃ R F a' b' c')`.
+{lean}`opaqueFix₃ R F` is the fixpoint of the functional {name}`F`, as long as the loop is
+well-founded.
+
+The loop might run forever depending on {name}`F`. It is opaque, i.e., it is impossible to prove
+nontrivial properties about it.
+-/
+@[specialize]
+public partial def opaqueFix₃ [∀ a b c, Nonempty (C₃ a b c)]
+    (R : (a : α) ×' (b : β a) ×' γ a b → (a : α) ×' (b : β a) ×' γ a b → Prop)
+    (F : ∀ (a b c), (∀ (a' b' c'), R ⟨a', b', c'⟩ ⟨a, b, c⟩ → C₃ a' b' c') → C₃ a b c)
+    (a : α) (b : β a) (c : γ a b) :
+    C₃ a b c :=
+  F a b c (fun a b c _ => opaqueFix₃ R F a b c)
+
+/-
+SAFE assuming that the code generated by iteration over `F` is equivalent to the well-founded
+fixpoint of `F` if `R` is well-founded.
+-/
+/--
+A fixpoint combinator that can be used to construct recursive functions of arity three with an
+*extrinsic* proof of termination.
+
+Given a relation {name}`R` and a fixpoint functional {name}`F` which must be decreasing with respect
+to {name}`R`, {lean}`extrinsicFix₃ R F` is the recursive function obtained by having {name}`F` call
+itself recursively.
+
+If {name}`R` is not well-founded, {lean}`extrinsicFix₃ R F` might run forever. In this case,
+nothing interesting can be proved about the recursive function; it is opaque.
+
+If {name}`R` _is_ well-founded, {lean}`extrinsicFix₃ R F` is equivalent to a well-founded recursive
+function, logically and regarding its termination behavior.
+-/
+@[implemented_by opaqueFix₃]
+public def extrinsicFix₃ [∀ a b c, Nonempty (C₃ a b c)]
+    (R : (a : α) ×' (b : β a) ×' γ a b → (a : α) ×' (b : β a) ×' γ a b → Prop)
+    (F : ∀ (a b c), (∀ (a' b' c'), R ⟨a', b', c'⟩ ⟨a, b, c⟩ → C₃ a' b' c') → C₃ a b c)
+    (a : α) (b : β a) (c : γ a b) :
+    C₃ a b c :=
+  let F' (x : (a : α) ×' (b : β a) ×' γ a b) (G : (y : (a : α) ×' (b : β a) ×' γ a b) → R y x → C₃ y.1 y.2.1 y.2.2) :
+      C₃ x.1 x.2.1 x.2.2 :=
+    F x.1 x.2.1 x.2.2 (fun a b c => G ⟨a, b, c⟩)
+  extrinsicFix (C := fun x : (a : α) ×' (b : β a) ×' γ a b => C₃ x.1 x.2.1 x.2.2) R F' ⟨a, b, c⟩
+
+public theorem extrinsicFix₃_eq_fix [∀ a b c, Nonempty (C₃ a b c)]
+    {R : (a : α) ×' (b : β a) ×' γ a b → (a : α) ×' (b : β a) ×' γ a b → Prop}
+    {F : ∀ a b c, (∀ a' b' c', R ⟨a', b', c'⟩ ⟨a, b, c⟩ → C₃ a' b' c') → C₃ a b c}
+    (wf : WellFounded R) {a b c} :
+    extrinsicFix₃ R F a b c = wf.fix (fun x G => F x.1 x.2.1 x.2.2 (fun a b c h => G ⟨a, b, c⟩ h)) ⟨a, b, c⟩ := by
+  rw [extrinsicFix₃, extrinsicFix_eq_fix wf]
+
+public theorem extrinsicFix₃_eq_apply [∀ a b c, Nonempty (C₃ a b c)]
+    {R : (a : α) ×' (b : β a) ×' γ a b → (a : α) ×' (b : β a) ×' γ a b → Prop}
+    {F : ∀ (a b c), (∀ (a' b' c'), R ⟨a', b', c'⟩ ⟨a, b, c⟩ → C₃ a' b' c') → C₃ a b c}
+    (wf : WellFounded R) {a : α} {b : β a} {c : γ a b} :
+    extrinsicFix₃ R F a b c = F a b c (fun a b c _ => extrinsicFix₃ R F a b c) := by
+  rw [extrinsicFix₃, extrinsicFix_eq_apply wf]
+  rfl
+
+/--
+A fixpoint combinator that allows for deferred proofs of termination.
+
+{lean}`extrinsicFix₃ R F` is function implemented as the loop
+{lean}`extrinsicFix₃ R F a b c = F a b c (fun a b c _ => extrinsicFix₃ R F a b c)`.
+
+If the loop can be shown to be well-founded, {name}`extrinsicFix₃_eq_apply` proves that it satisfies the
+fixpoint equation. Otherwise, {lean}`extrinsicFix₃ R F` is opaque, i.e., it is impossible to prove
+nontrivial properties about it.
+-/
+add_decl_doc extrinsicFix₃
+
+end WellFounded

src/Lean/Attributes.lean

@@ -250,7 +250,7 @@ structure ParametricAttributeImpl (α : Type) extends AttributeImplCore where
   afterSet : Name → α → AttrM Unit := fun _ _ _ => pure ()
   /--
   If set, entries are not resorted on export and `getParam?` will fall back to a linear instead of
-  binary search insde an imported module's entries.
+  binary search inside an imported module's entries.
   -/
   preserveOrder : Bool := false
   /--

src/Lean/Compiler/IR/Boxing.lean

@@ -53,7 +53,7 @@ def mkBoxedVersionAux (decl : Decl) : N Decl := do
     pure <| reshape newVDecls (.ret (.var r))
   else
     let newR ← N.mkFresh
-    let newVDecls := newVDecls.push (.vdecl newR .tobject (.box decl.resultType r) default)
+    let newVDecls := newVDecls.push (.vdecl newR decl.resultType.boxed (.box decl.resultType r) default)
     pure <| reshape newVDecls (.ret (.var newR))
   return Decl.fdecl (mkBoxedName decl.name) qs decl.resultType.boxed body decl.getInfo
 
@@ -267,8 +267,29 @@ def visitVDeclExpr (x : VarId) (ty : IRType) (e : Expr) (b : FnBody) : M FnBody
   | _     =>
     return .vdecl x ty e b
 
+/--
+Up to this point the type system of IR is quite loose so we can for example encounter situations
+such as
+```
+let y : obj := f x
+```
+where `f : obj -> uint8`. It is the job of the boxing pass to enforce a stricter obj/scalar
+separation and as such it needs to correct situations like this.
+-/
+def tryCorrectVDeclType (ty : IRType) (e : Expr) : M IRType :=
+  match e with
+  | .fap f _ => do
+    let decl ← getDecl f
+    return decl.resultType
+  | .pap .. => return .object
+  | .uproj .. => return .usize
+  | .ctor .. | .reuse .. | .ap .. | .lit .. | .sproj .. | .proj .. | .reset .. =>
+    return ty
+  | .unbox .. | .box .. | .isShared .. => unreachable!
+
 partial def visitFnBody : FnBody → M FnBody
   | .vdecl x t v b     => do
+    let t ← tryCorrectVDeclType t v
     let b ← withVDecl x t v (visitFnBody b)
     visitVDeclExpr x t v b
   | .jdecl j xs v b    => do

src/Lean/Compiler/IR/LiveVars.lean

@@ -68,7 +68,7 @@ partial def visitFnBody (w : Index) : FnBody → M Bool
         -- Instead of marking the join points that we have already been visited, we permanently remove `j` from the context.
         set (ctx.eraseJoinPointDecl j) *> visitFnBody w b
       | none   =>
-        -- `j` must be a local join point. So do nothing since we have already visite its body.
+        -- `j` must be a local join point. So do nothing since we have already visited its body.
         pure false
   | .ret x =>
     visitArg w x

src/Lean/Compiler/IR/ToIR.lean

@@ -19,6 +19,14 @@ open Lean.Compiler (LCNF.Alt LCNF.Arg LCNF.Code LCNF.Decl LCNF.DeclValue LCNF.LC
 
 namespace ToIR
 
+/--
+Marks an extern definition to be guaranteed to always return tagged values.
+This information is used to optimize reference counting in the compiler.
+-/
+@[builtin_doc]
+builtin_initialize taggedReturnAttr : TagAttribute ←
+  registerTagAttribute `tagged_return "mark extern definition to always return tagged values"
+
 structure BuilderState where
   vars : Std.HashMap FVarId Arg := {}
   joinPoints : Std.HashMap FVarId JoinPointId := {}
@@ -334,12 +342,20 @@ where resultTypeForArity (type : Lean.Expr) (arity : Nat) : Lean.Expr :=
 
 def lowerDecl (d : LCNF.Decl) : M (Option Decl) := do
   let params ← d.params.mapM lowerParam
-  let resultType ← lowerResultType d.type d.params.size
+  let mut resultType ← lowerResultType d.type d.params.size
+  let taggedReturn := taggedReturnAttr.hasTag (← getEnv) d.name
   match d.value with
   | .code code =>
+    if taggedReturn then
+      throwError m!"Error while compiling function '{d.name}': @[tagged_return] is only valid for extern declarations"
     let body ← lowerCode code
     pure <| some <| .fdecl d.name params resultType body {}
   | .extern externAttrData =>
+    if taggedReturn then
+      if resultType.isScalar then
+        throwError m!"@[tagged_return] on function '{d.name}' with scalar return type {resultType}"
+      else
+        resultType := .tagged
     if externAttrData.entries.isEmpty then
       -- TODO: This matches the behavior of the old compiler, but we should
       -- find a better way to handle this.

src/Lean/Compiler/InlineAttrs.lean

@@ -37,6 +37,7 @@ private def isValidMacroInline (declName : Name) : CoreM Bool := do
   let isRec (declName' : Name) : Bool :=
     isBRecOnRecursor env declName' ||
     declName' == ``WellFounded.fix ||
+    declName' == ``WellFounded.Nat.fix ||
     declName' == declName ++ `_unary -- Auxiliary declaration created by `WF` module
   if Option.isSome <| info.value.find? fun e => e.isConst && isRec e.constName! then
     -- It contains a `brecOn` or `WellFounded.fix` application. So, it should be recursvie

src/Lean/Compiler/LCNF/LambdaLifting.lean

@@ -40,6 +40,10 @@ structure Context where
   We use this feature to implement `@[inline] instance ...` and `@[always_inline] instance ...`
   -/
   minSize : Nat := 0
+  /--
+  Allow for eta contraction instead of lifting to a lambda if possible.
+  -/
+  allowEtaContraction : Bool := true
 
 
 /-- State for the `LiftM` monad. -/
@@ -81,6 +85,13 @@ partial def mkAuxDeclName : LiftM Name := do
   if (← getDecl? nameNew).isNone then return nameNew
   mkAuxDeclName
 
+def replaceFunDecl (decl : FunDecl) (value : LetValue) : LiftM LetDecl := do
+  /- We reuse `decl`s `fvarId` to avoid substitution -/
+  let declNew := { fvarId := decl.fvarId, binderName := decl.binderName, type := decl.type, value }
+  modifyLCtx fun lctx => lctx.addLetDecl declNew
+  eraseFunDecl decl
+  return declNew
+
 open Internalize in
 /--
 Create a new auxiliary declaration. The array `closure` contains all free variables
@@ -100,11 +111,7 @@ def mkAuxDecl (closure : Array Param) (decl : FunDecl) : LiftM LetDecl := do
     auxDecl.erase
     pure declName
   let value := .const auxDeclName us (closure.map (.fvar ·.fvarId))
-  /- We reuse `decl`s `fvarId` to avoid substitution -/
-  let declNew := { fvarId := decl.fvarId, binderName := decl.binderName, type := decl.type, value }
-  modifyLCtx fun lctx => lctx.addLetDecl declNew
-  eraseFunDecl decl
-  return declNew
+  replaceFunDecl decl value
 where
   go (nameNew : Name) (safe : Bool) (inlineAttr? : Option InlineAttributeKind) : InternalizeM Decl := do
     let params := (← closure.mapM internalizeParam) ++ (← decl.params.mapM internalizeParam)
@@ -115,6 +122,20 @@ where
     let decl := { name := nameNew, levelParams := [], params, type, value, safe, inlineAttr?, recursive := false : Decl }
     return decl.setLevelParams
 
+def etaContractibleDecl? (decl : FunDecl) : LiftM (Option LetDecl) := do
+  if !(← read).allowEtaContraction then return none
+  let .let { fvarId := letVar, value := .const declName us args, .. } (.return retVar) := decl.value
+    | return none
+  if letVar != retVar then return none
+  if args.size != decl.params.size then return none
+  if (← getDecl? declName).isNone then return none
+  for arg in args, param in decl.params do
+    let .fvar argVar := arg | return none
+    if argVar != param.fvarId then return none
+
+  let value := .const declName us #[]
+  replaceFunDecl decl value
+
 mutual
   partial def visitFunDecl (funDecl : FunDecl) : LiftM FunDecl := do
     let value ← withParams funDecl.params <| visitCode funDecl.value
@@ -128,9 +149,13 @@ mutual
     | .fun decl k =>
       let decl ← visitFunDecl decl
       if (← shouldLift decl) then
-        let scope ← getScope
-        let (_, params, _) ← Closure.run (inScope := scope.contains) <| Closure.collectFunDecl decl
-        let declNew ← mkAuxDecl params decl
+        let declNew ← do
+          if let some letDecl ← etaContractibleDecl? decl then
+            pure letDecl
+          else
+            let scope ← getScope
+            let (_, params, _) ← Closure.run (inScope := scope.contains) <| Closure.collectFunDecl decl
+            mkAuxDecl params decl
         let k ← withFVar declNew.fvarId <| visitCode k
         return .let declNew k
       else
@@ -155,8 +180,17 @@ def main (decl : Decl) : LiftM Decl := do
 
 end LambdaLifting
 
-partial def Decl.lambdaLifting (decl : Decl) (liftInstParamOnly : Bool) (suffix : Name) (inheritInlineAttrs := false) (minSize := 0) : CompilerM (Array Decl) := do
-  let (decl, s) ← LambdaLifting.main decl |>.run { mainDecl := decl, liftInstParamOnly, suffix, inheritInlineAttrs, minSize } |>.run {} |>.run {}
+partial def Decl.lambdaLifting (decl : Decl) (liftInstParamOnly : Bool) (allowEtaContraction : Bool)
+    (suffix : Name) (inheritInlineAttrs := false) (minSize := 0) : CompilerM (Array Decl) := do
+  let ctx := {
+    mainDecl := decl,
+    liftInstParamOnly,
+    suffix,
+    inheritInlineAttrs,
+    minSize,
+    allowEtaContraction
+  }
+  let (decl, s) ← LambdaLifting.main decl |>.run ctx |>.run {} |>.run {}
   return s.decls.push decl
 
 /--
@@ -166,7 +200,8 @@ def lambdaLifting : Pass where
   phase      := .mono
   name       := `lambdaLifting
   run        := fun decls => do
-    decls.foldlM (init := #[]) fun decls decl => return decls ++ (← decl.lambdaLifting false (suffix := `_lam))
+    decls.foldlM (init := #[]) fun decls decl =>
+      return decls ++ (← decl.lambdaLifting false true (suffix := `_lam))
 
 /--
 During eager lambda lifting, we inspect declarations that are not inlineable or instances (doing it
@@ -182,7 +217,7 @@ def eagerLambdaLifting : Pass where
       if decl.inlineable || (← Meta.isInstance decl.name) then
         return decls.push decl
       else
-        return decls ++ (← decl.lambdaLifting (liftInstParamOnly := true) (suffix := `_elam))
+        return decls ++ (← decl.lambdaLifting (liftInstParamOnly := true) (allowEtaContraction := false) (suffix := `_elam))
 
 builtin_initialize
   registerTraceClass `Compiler.eagerLambdaLifting (inherited := true)

src/Lean/Compiler/LCNF/Simp/ConstantFold.lean

@@ -372,6 +372,7 @@ def arithmeticFolders : List (Name × Folder) := [
   (``UInt64.sub,  Folder.first #[Folder.mkBinary UInt64.sub, Folder.rightNeutral (0 : UInt64) (· - ·)]),
   -- We don't convert Nat multiplication by a power of 2 into a left shift, because the fast path
   -- for multiplication isn't any slower than a fast path for left shift that checks for overflow.
+  (``Nat.mul, Folder.first #[Folder.mkBinary Nat.mul, Folder.leftRightNeutral (1 : Nat) (· * ·), Folder.leftRightAnnihilator (0 : Nat) 0 (· * ·)]),
   (``UInt8.mul,  Folder.first #[Folder.mkBinary UInt8.mul, Folder.leftRightNeutral (1 : UInt8) (· * ·), Folder.leftRightAnnihilator (0 : UInt8) 0 (· * ·), Folder.mulShift ``UInt8.shiftLeft (UInt8.shiftLeft 1 ·) UInt8.log2]),
   (``UInt16.mul,  Folder.first #[Folder.mkBinary UInt16.mul, Folder.leftRightNeutral (1 : UInt16) (· * ·), Folder.leftRightAnnihilator (0 : UInt16) 0 (· * ·), Folder.mulShift ``UInt16.shiftLeft (UInt16.shiftLeft 1 ·) UInt16.log2]),
   (``UInt32.mul,  Folder.first #[Folder.mkBinary UInt32.mul, Folder.leftRightNeutral (1 : UInt32) (· * ·), Folder.leftRightAnnihilator (0 : UInt32) 0 (· * ·), Folder.mulShift ``UInt32.shiftLeft (UInt32.shiftLeft 1 ·) UInt32.log2]),

src/Lean/Compiler/LCNF/Simp/Main.lean

@@ -336,6 +336,18 @@ partial def simp (code : Code) : SimpM Code := withIncRecDepth do
               params.forM (eraseParam ·)
               markSimplified
               return k
+        if alts.all (·.getCode matches .unreach ..) then
+          alts.forM (liftM <| ·.getParams.forM eraseParam)
+          markSimplified
+          return .unreach resultType
+        /-
+        We considered handling a case where we drop a cases if it only has one non-unreachable
+        branch and doesn't rely on the params of that branch here. However, this has the potential
+        to hinder reuse in later passes as we loose information about the shape of a variable. We
+        might be able to reintroduce this at a later point if we track this information different
+        from cases.
+        -/
+
         markUsedFVar discr
         return code.updateCases! resultType discr alts
 end

src/Lean/Compiler/LCNF/ToLCNF.lean

@@ -666,11 +666,11 @@ where
     let .const declName _ := e.getAppFn | unreachable!
     let typeName := declName.getPrefix
     let .inductInfo inductVal ← getConstInfo typeName | unreachable!
-    let arity := inductVal.numParams + inductVal.numIndices + 1 /- motive -/ + 2 /- lhs/rhs-/ + 1 /- equality -/
+    let arity := inductVal.numParams + 1 /- motive -/ + 3*(inductVal.numIndices + 1) /- lhs/rhs and equalities -/
     etaIfUnderApplied e arity do
       let args := e.getAppArgs
-      let lhs ← liftMetaM do Meta.whnf args[inductVal.numParams + inductVal.numIndices + 1]!
-      let rhs ← liftMetaM do Meta.whnf args[inductVal.numParams + inductVal.numIndices + 2]!
+      let lhs ← liftMetaM do Meta.whnf args[inductVal.numParams + 1 + inductVal.numIndices]!
+      let rhs ← liftMetaM do Meta.whnf args[inductVal.numParams + 1 + inductVal.numIndices + 1 + inductVal.numIndices]!
       let lhs ← liftMetaM lhs.toCtorIfLit
       let rhs ← liftMetaM rhs.toCtorIfLit
       match (← liftMetaM <| Meta.isConstructorApp? lhs), (← liftMetaM <| Meta.isConstructorApp? rhs) with

src/Lean/Data/Json/FromToJson/Basic.lean

@@ -46,6 +46,8 @@ instance : FromJson Int := ⟨Json.getInt?⟩
 instance : ToJson Int := ⟨fun n => Json.num n⟩
 instance : FromJson String := ⟨Json.getStr?⟩
 instance : ToJson String := ⟨fun s => s⟩
+instance : FromJson String.Slice := ⟨Except.map String.toSlice ∘ Json.getStr?⟩
+instance : ToJson String.Slice := ⟨fun s => s.copy⟩
 
 instance : FromJson System.FilePath := ⟨fun j => System.FilePath.mk <$> Json.getStr? j⟩
 instance : ToJson System.FilePath := ⟨fun p => p.toString⟩

src/Lean/Data/Lsp/Extra.lean

@@ -63,8 +63,8 @@ an ILean finalization notification for the worker and the document version desig
 Used for test stability in tests that use the .ileans.
 -/
 structure WaitForILeansParams where
-  uri     : DocumentUri
-  version : Nat
+  uri?     : Option DocumentUri := none
+  version? : Option Nat := none
   deriving FromJson, ToJson
 
 structure WaitForILeans where

src/Lean/Data/Lsp/Internal.lean

@@ -10,6 +10,7 @@ prelude
 public import Lean.Expr
 public import Lean.Data.Lsp.Basic
 public import Lean.Data.JsonRpc
+public import Lean.Data.DeclarationRange
 
 public section
 
@@ -98,27 +99,129 @@ instance : ToJson RefIdent where
 
 end RefIdent
 
-/-- Information about the declaration surrounding a reference. -/
-structure RefInfo.ParentDecl where
-  /-- Name of the declaration surrounding a reference. -/
-  name           : String
-  /-- Range of the declaration surrounding a reference. -/
-  range          : Lsp.Range
-  /-- Selection range of the declaration surrounding a reference. -/
-  selectionRange : Lsp.Range
-  deriving ToJson
+/-- Position information for a declaration. Inlined to reduce memory consumption. -/
+structure DeclInfo where
+  /-- Start line of range. -/
+  rangeStartPosLine : Nat
+  /-- Start character of range. -/
+  rangeStartPosCharacter : Nat
+  /-- End line of range. -/
+  rangeEndPosLine : Nat
+  /-- End character of range. -/
+  rangeEndPosCharacter : Nat
+  /-- Start line of selection range. -/
+  selectionRangeStartPosLine : Nat
+  /-- Start character of selection range. -/
+  selectionRangeStartPosCharacter : Nat
+  /-- End line of selection range. -/
+  selectionRangeEndPosLine : Nat
+  /-- End character of selection range. -/
+  selectionRangeEndPosCharacter : Nat
+
+/-- Converts a set of `DeclarationRanges` to a `DeclInfo`. -/
+def DeclInfo.ofDeclarationRanges (r : DeclarationRanges) : DeclInfo where
+  rangeStartPosLine := r.range.pos.line - 1
+  rangeStartPosCharacter := r.range.charUtf16
+  rangeEndPosLine := r.range.endPos.line - 1
+  rangeEndPosCharacter := r.range.endCharUtf16
+  selectionRangeStartPosLine := r.selectionRange.pos.line - 1
+  selectionRangeStartPosCharacter := r.selectionRange.charUtf16
+  selectionRangeEndPosLine := r.selectionRange.endPos.line - 1
+  selectionRangeEndPosCharacter := r.selectionRange.endCharUtf16
+
+/-- Range of this parent decl. -/
+def DeclInfo.range (i : DeclInfo) : Lsp.Range :=
+  ⟨⟨i.rangeStartPosLine, i.rangeStartPosCharacter⟩, ⟨i.rangeEndPosLine, i.rangeEndPosCharacter⟩⟩
+
+/-- Selection range of this parent decl. -/
+def DeclInfo.selectionRange (i : DeclInfo) : Lsp.Range :=
+  ⟨⟨i.selectionRangeStartPosLine, i.selectionRangeStartPosCharacter⟩,
+    ⟨i.selectionRangeEndPosLine, i.selectionRangeEndPosCharacter⟩⟩
+
+instance : ToJson DeclInfo where
+  toJson i :=
+    Json.arr #[
+      i.rangeStartPosLine,
+      i.rangeStartPosCharacter,
+      i.rangeEndPosLine,
+      i.rangeEndPosCharacter,
+      i.selectionRangeStartPosLine,
+      i.selectionRangeStartPosCharacter,
+      i.selectionRangeEndPosLine,
+      i.selectionRangeEndPosCharacter
+    ]
+
+instance : FromJson DeclInfo where
+  fromJson?
+    | .arr xs => do
+      if xs.size != 8 then
+        throw s!"Expected list of length 8, not length {xs.size}"
+      return {
+        rangeStartPosLine := ← fromJson? xs[0]!
+        rangeStartPosCharacter := ← fromJson? xs[1]!
+        rangeEndPosLine := ← fromJson? xs[2]!
+        rangeEndPosCharacter := ← fromJson? xs[3]!
+        selectionRangeStartPosLine := ← fromJson? xs[4]!
+        selectionRangeStartPosCharacter := ← fromJson? xs[5]!
+        selectionRangeEndPosLine := ← fromJson? xs[6]!
+        selectionRangeEndPosCharacter := ← fromJson? xs[7]!
+      }
+    | _ => throw "Expected list"
+
+/-- Declarations of a file with associated position information. -/
+@[expose] def Decls := Std.TreeMap String DeclInfo
+  deriving EmptyCollection, ForIn Id
+
+instance : ToJson Decls where
+  toJson m := Json.mkObj <| m.toList.map fun (declName, info) => (declName, toJson info)
+
+instance : FromJson Decls where
+  fromJson? j := do
+    let node ← j.getObj?
+    node.foldlM (init := ∅) fun m k v =>
+      return m.insert k (← fromJson? v)
 
 /--
 Denotes the range of a reference, as well as the parent declaration of the reference.
 If the reference is itself a declaration, then it contains no parent declaration.
+The position information is inlined to reduce memory consumption.
 -/
 structure RefInfo.Location where
-  /-- Range of the reference. -/
-  range       : Lsp.Range
-  /-- Parent declaration of the reference. `none` if the reference is itself a declaration. -/
-  parentDecl? : Option RefInfo.ParentDecl
+  mk' ::
+  /-- Start line of the range of this location. -/
+  startPosLine : Nat
+  /-- Start character of the range of this location. -/
+  startPosCharacter : Nat
+  /-- End line of the range of this location. -/
+  endPosLine : Nat
+  /-- End character of the range of this location. -/
+  endPosCharacter : Nat
+  /--
+  Parent declaration of the reference. Empty string if the reference is itself a declaration.
+  We do not use `Option` for memory consumption reasons.
+  -/
+  parentDecl : String
 deriving Inhabited
 
+/-- Creates a `RefInfo.Location`. -/
+def RefInfo.Location.mk (range : Lsp.Range) (parentDecl? : Option String) : RefInfo.Location where
+  startPosLine := range.start.line
+  startPosCharacter := range.start.character
+  endPosLine := range.end.line
+  endPosCharacter := range.end.character
+  parentDecl := parentDecl?.getD ""
+
+/-- Range of this location. -/
+def RefInfo.Location.range (l : RefInfo.Location) : Lsp.Range :=
+  ⟨⟨l.startPosLine, l.startPosCharacter⟩, ⟨l.endPosLine, l.endPosCharacter⟩⟩
+
+/-- Name of the parent declaration of this location. -/
+def RefInfo.Location.parentDecl? (l : RefInfo.Location) : Option String :=
+  if l.parentDecl.isEmpty then
+    none
+  else
+    some l.parentDecl
+
 /-- Definition site and usage sites of a reference. Obtained from `Lean.Server.RefInfo`. -/
 structure RefInfo where
   /-- Definition site of the reference. May be `none` when we cannot find a definition site. -/
@@ -128,16 +231,9 @@ structure RefInfo where
 
 instance : ToJson RefInfo where
   toJson i :=
-    let rangeToList (r : Lsp.Range) : List Nat :=
-      [r.start.line, r.start.character, r.end.line, r.end.character]
-    let parentDeclToList (d : RefInfo.ParentDecl) : List Json :=
-      let name := d.name |> toJson
-      let range := rangeToList d.range |>.map toJson
-      let selectionRange := rangeToList d.selectionRange |>.map toJson
-      [name] ++ range ++ selectionRange
     let locationToList (l : RefInfo.Location) : List Json :=
-      let range := rangeToList l.range |>.map toJson
-      let parentDecl := l.parentDecl?.map parentDeclToList |>.getD []
+      let range := [l.startPosLine, l.startPosCharacter, l.endPosLine, l.endPosCharacter].map toJson
+      let parentDecl := l.parentDecl?.map ([toJson ·]) |>.getD []
       range ++ parentDecl
     Json.mkObj [
       ("definition", toJson $ i.definition?.map locationToList),
@@ -147,35 +243,30 @@ instance : ToJson RefInfo where
 instance : FromJson RefInfo where
   -- This implementation is optimized to prevent redundant intermediate allocations.
   fromJson? j := do
-    let toRange (a : Array Json) (i : Nat) : Except String Lsp.Range :=
-      if h : a.size < i + 4 then
-        throw s!"Expected list of length 4, not {a.size}"
-      else
-        return {
-          start := {
-            line := ← fromJson? a[i]
-            character := ← fromJson? a[i+1]
-          }
-          «end» := {
-            line := ← fromJson? a[i+2]
-            character := ← fromJson? a[i+3]
-          }
-        }
-    let toParentDecl (a : Array Json) (i : Nat) : Except String RefInfo.ParentDecl := do
-      let name ← fromJson? a[i]!
-      let range ← toRange a (i + 1)
-      let selectionRange ← toRange a (i + 5)
-      return ⟨name, range, selectionRange⟩
     let toLocation (a : Array Json) : Except String RefInfo.Location := do
-      if a.size != 4 && a.size != 13 then
-        .error "Expected list of length 4 or 13, not {l.size}"
-      let range ← toRange a 0
-      if a.size == 13 then
-        let parentDecl ← toParentDecl a 4
-        return ⟨range, parentDecl⟩
+      if h : a.size ≠ 4 ∧ a.size ≠ 5 then
+        .error s!"Expected list of length 4 or 5, not {a.size}"
       else
-        return ⟨range, none⟩
-
+        let startPosLine ← fromJson? a[0]
+        let startPosCharacter ← fromJson? a[1]
+        let endPosLine ← fromJson? a[2]
+        let endPosCharacter ← fromJson? a[3]
+        if h' : a.size = 5 then
+          return {
+            startPosLine
+            startPosCharacter
+            endPosLine
+            endPosCharacter
+            parentDecl := ← fromJson? a[4]
+          }
+        else
+          return {
+            startPosLine
+            startPosCharacter
+            endPosLine
+            endPosCharacter
+            parentDecl := ""
+          }
     let definition? ← j.getObjValAs? (Option $ Array Json) "definition"
     let definition? ← match definition? with
       | none => pure none
@@ -213,15 +304,28 @@ structure LeanILeanHeaderInfoParams where
   directImports : Array ImportInfo
   deriving FromJson, ToJson
 
+/--
+Used in the `$/lean/ileanHeaderSetupInfo` watchdog <- worker notifications.
+Contains status information about `lake setup-file`.
+-/
+structure LeanILeanHeaderSetupInfoParams where
+  /-- Version of the file these imports are from. -/
+  version        : Nat
+  /-- Whether setting up the header using `lake setup-file` has failed. -/
+  isSetupFailure : Bool
+  deriving FromJson, ToJson
+
 /--
 Used in the `$/lean/ileanInfoUpdate` and `$/lean/ileanInfoFinal` watchdog <- worker notifications.
 Contains the definitions and references of the file managed by a worker.
 -/
 structure LeanIleanInfoParams where
   /-- Version of the file these references are from. -/
-  version        : Nat
+  version    : Nat
   /-- All references for the file. -/
-  references     : ModuleRefs
+  references : ModuleRefs
+  /-- All decls for the file. -/
+  decls      : Decls
   deriving FromJson, ToJson
 
 /--

src/Lean/Data/Lsp/Ipc.lean

@@ -141,6 +141,19 @@ partial def waitForILeans (waitForILeansId : RequestID := 0) (target : DocumentU
     | _ =>
       pure ()
 
+partial def waitForWatchdogILeans (waitForILeansId : RequestID := 0) : IpcM Unit := do
+  writeRequest ⟨waitForILeansId, "$/lean/waitForILeans", WaitForILeansParams.mk none none⟩
+  while true do
+    match (← readMessage) with
+    | .response id _ =>
+      if id == waitForILeansId then
+        return
+    | .responseError id _ msg _ =>
+      if id == waitForILeansId then
+        throw $ userError s!"Waiting for ILeans failed: {msg}"
+    | _ =>
+      pure ()
+
 /--
 Waits for a diagnostic notification with a specific message to be emitted. Discards all received
 messages, so should not be combined with `collectDiagnostics`.

src/Lean/Data/PrefixTree.lean

@@ -52,7 +52,7 @@ partial def find? (cmp : α → α → Ordering) (t : PrefixTreeNode α β cmp)
       | some t => loop t ks
   loop t k
 
-/-- Returns the the value of the longest key in `t` that is a prefix of `k`, if any. -/
+/-- Returns the value of the longest key in `t` that is a prefix of `k`, if any. -/
 @[inline]
 partial def findLongestPrefix? (cmp : α → α → Ordering) (t : PrefixTreeNode α β cmp) (k : List α) : Option β :=
   let rec @[specialize] loop acc?

src/Lean/Data/Xml/Parser.lean

@@ -154,7 +154,7 @@ def SystemLiteral : Parser String :=
 /-- https://www.w3.org/TR/xml/#NT-PubidChar -/
 def PubidChar : Parser LeanChar :=
   asciiLetter <|> digit <|> endl <|> attempt do
-  let c ← any
+  let c : _root_.Char := ← any
   if "-'()+,./:=?;!*#@$_%".contains c then pure c else fail "PublidChar expected"
 
 /-- https://www.w3.org/TR/xml/#NT-PubidLiteral -/

src/Lean/DocString/Links.lean

@@ -124,7 +124,7 @@ def rewriteManualLinksCore (s : String) : Id (Array (Lean.Syntax.Range × String
       iter' := iter'.next h'
       if urlChar c' && ¬iter'.IsAtEnd then
         continue
-      match rw (start.extract pre') with
+      match rw (s.extract start pre') with
       | .error err =>
         errors := errors.push (⟨pre.offset, pre'.offset⟩, err)
         out := out.push c