.

.github/workflows/ci.yml

@@ -244,21 +244,21 @@ jobs:
                 "check-level": 2,
                 "cross": true,
                 "shell": "bash -euxo pipefail {0}"
+              },
+              {
+                "name": "Web Assembly",
+                "os": "ubuntu-latest",
+                // Build a native 32bit binary in stage0 and use it to compile the oleans and the wasm build
+                "CMAKE_OPTIONS": "-DCMAKE_C_COMPILER_WORKS=1 -DSTAGE0_USE_GMP=OFF -DSTAGE0_LEAN_EXTRA_CXX_FLAGS='-m32' -DSTAGE0_LEANC_OPTS='-m32' -DSTAGE0_CMAKE_CXX_COMPILER=clang++ -DSTAGE0_CMAKE_C_COMPILER=clang -DSTAGE0_CMAKE_EXECUTABLE_SUFFIX=\"\" -DUSE_GMP=OFF -DMMAP=OFF -DSTAGE0_MMAP=OFF -DCMAKE_AR=../emsdk/emsdk-main/upstream/emscripten/emar -DCMAKE_TOOLCHAIN_FILE=../emsdk/emsdk-main/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake -DLEAN_INSTALL_SUFFIX=-linux_wasm32 -DSTAGE0_CMAKE_LIBRARY_PATH=/usr/lib/i386-linux-gnu/",
+                "wasm": true,
+                "cmultilib": true,
+                "release": true,
+                "check-level": 2,
+                "cross": true,
+                "shell": "bash -euxo pipefail {0}",
+                // Just a few selected tests because wasm is slow
+                "CTEST_OPTIONS": "-R \"leantest_1007\\.lean|leantest_Format\\.lean|leanruntest\\_1037.lean|leanruntest_ac_rfl\\.lean|leanruntest_tempfile.lean\\.|leanruntest_libuv\\.lean\""
               }
-              // {
-              //   "name": "Web Assembly",
-              //   "os": "ubuntu-latest",
-              //   // Build a native 32bit binary in stage0 and use it to compile the oleans and the wasm build
-              //   "CMAKE_OPTIONS": "-DCMAKE_C_COMPILER_WORKS=1 -DSTAGE0_USE_GMP=OFF -DSTAGE0_LEAN_EXTRA_CXX_FLAGS='-m32' -DSTAGE0_LEANC_OPTS='-m32' -DSTAGE0_CMAKE_CXX_COMPILER=clang++ -DSTAGE0_CMAKE_C_COMPILER=clang -DSTAGE0_CMAKE_EXECUTABLE_SUFFIX=\"\" -DUSE_GMP=OFF -DMMAP=OFF -DSTAGE0_MMAP=OFF -DCMAKE_AR=../emsdk/emsdk-main/upstream/emscripten/emar -DCMAKE_TOOLCHAIN_FILE=../emsdk/emsdk-main/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake -DLEAN_INSTALL_SUFFIX=-linux_wasm32 -DSTAGE0_CMAKE_LIBRARY_PATH=/usr/lib/i386-linux-gnu/",
-              //   "wasm": true,
-              //   "cmultilib": true,
-              //   "release": true,
-              //   "check-level": 2,
-              //   "cross": true,
-              //   "shell": "bash -euxo pipefail {0}",
-              //   // Just a few selected tests because wasm is slow
-              //   "CTEST_OPTIONS": "-R \"leantest_1007\\.lean|leantest_Format\\.lean|leanruntest\\_1037.lean|leanruntest_ac_rfl\\.lean|leanruntest_tempfile.lean\\.|leanruntest_libuv\\.lean\""
-              // }
             ];
             console.log(`matrix:\n${JSON.stringify(matrix, null, 2)}`)
             return matrix.filter((job) => level >= job["check-level"])

README.md

@@ -6,8 +6,7 @@ This is the repository for **Lean 4**.
 - [Homepage](https://lean-lang.org)
 - [Theorem Proving Tutorial](https://lean-lang.org/theorem_proving_in_lean4/)
 - [Functional Programming in Lean](https://lean-lang.org/functional_programming_in_lean/)
-- [Documentation Overview](https://lean-lang.org/lean4/doc/)
-- [Language Reference](https://lean-lang.org/doc/reference/latest/)
+- [Manual](https://lean-lang.org/lean4/doc/)
 - [Release notes](RELEASES.md) starting at v4.0.0-m3
 - [Examples](https://lean-lang.org/lean4/doc/examples.html)
 - [External Contribution Guidelines](CONTRIBUTING.md)

doc/SUMMARY.md

@@ -13,13 +13,61 @@
   - [The Well-Typed Interpreter](examples/interp.lean.md)
   - [Dependent de Bruijn Indices](examples/deBruijn.lean.md)
   - [Parametric Higher-Order Abstract Syntax](examples/phoas.lean.md)
-  - [Syntax Examples](./syntax_examples.md)
-    - [Balanced Parentheses](./syntax_example.md)
-    - [Arithmetic DSL](./metaprogramming-arith.md)
 
 # Language Manual
-
-- [The Lean Reference Manual](./reference.md)
+<!-- - [Using Lean](./using_lean.md) -->
+<!-- - [Lexical Structure](./lexical_structure.md) -->
+<!-- - [Expressions](./expressions.md) -->
+<!-- - [Declarations](./declarations.md) -->
+- [Organizational features](./organization.md)
+  - [Sections](./sections.md)
+  - [Namespaces](./namespaces.md)
+  - [Implicit Arguments](./implicit.md)
+  - [Auto Bound Implicit Arguments](./autobound.md)
+<!-- - [Dependent Types](./deptypes.md) -->
+<!--   - [Simple Type Theory](./simptypes.md) -->
+<!--   - [Types as objects](./typeobjs.md) -->
+<!--   - [Function Abstraction and Evaluation](./funabst.md) -->
+<!--   - [Introducing Definitions](./introdef.md) -->
+<!--   - [What makes dependent type theory dependent?](./dep.md) -->
+<!-- - [Tactics](./tactics.md) -->
+- [Syntax Extensions](./syntax.md)
+  - [The `do` Notation](./do.md)
+  - [String Interpolation](./stringinterp.md)
+  - [User-Defined Notation](./notation.md)
+  - [Macro Overview](./macro_overview.md)
+  - [Elaborators](./elaborators.md)
+  - [Examples](./syntax_examples.md)
+    - [Balanced Parentheses](./syntax_example.md)
+    - [Arithmetic DSL](./metaprogramming-arith.md)
+- [Declaring New Types](./decltypes.md)
+  - [Enumerated Types](./enum.md)
+  - [Inductive Types](./inductive.md)
+  - [Structures](./struct.md)
+  - [Type classes](./typeclass.md)
+  - [Unification Hints](./unifhint.md)
+- [Builtin Types](./builtintypes.md)
+  - [Natural number](./nat.md)
+  - [Integer](./int.md)
+  - [Fixed precision unsigned integer](./uint.md)
+  - [Float](./float.md)
+  - [Array](./array.md)
+  - [List](./list.md)
+  - [Character](./char.md)
+  - [String](./string.md)
+  - [Option](./option.md)
+  - [Thunk](./thunk.md)
+  - [Task and Thread](./task.md)
+- [Functions](./functions.md)
+- [Monads](./monads/intro.md)
+  - [Functor](./monads/functors.lean.md)
+  - [Applicative](./monads/applicatives.lean.md)
+  - [Monad](./monads/monads.lean.md)
+  - [Reader](./monads/readers.lean.md)
+  - [State](./monads/states.lean.md)
+  - [Except](./monads/except.lean.md)
+  - [Transformers](./monads/transformers.lean.md)
+  - [Laws](./monads/laws.lean.md)
 
 # Other
 

doc/array.md

@@ -0,0 +1,77 @@
+# Arrays
+
+The `Array` type implements a *dynamic* (aka growable) array.
+It is defined as
+```lean
+# namespace hidden
+structure Array (α : Type u) where
+  data : List α
+# end hidden
+```
+but its execution time representation is optimized, and it is similar to C++ `std::vector<T>` and Rust `Vec<T>`.
+The Lean type checker has no special support for reducing `Array`s.
+
+You can create arrays in several ways. You can create a small array by listing consecutive values between
+`#[` and `]` and separated by commas, as shown in the following examples.
+
+```lean
+#check #[1, 2, 3] -- Array Nat
+
+#check #[] -- Array ?m
+```
+
+The type of the array elements is inferred from the literals used and must be consistent.
+```lean
+#check #["hello", "world"] -- Array String
+
+-- The following is not valid
+#check_failure #[10, "hello"]
+```
+Recall that the command `#check_failure <term>` only succeeds when the given term is not type correct.
+
+To create an array of size `n` in which all the elements are initialized to some value `a`, use `mkArray`.
+```lean
+#eval mkArray 5 'a'
+-- #['a', 'a', 'a', 'a', 'a']
+```
+
+## Accessing elements
+
+You can access array elements by using brackets (`[` and `]`).
+```lean
+def f (a : Array Nat) (i : Fin a.size) :=
+  a[i] + a[i]
+```
+Note that the index `i` has type `Fin a.size`, i.e., it is natural number less than `a.size`.
+You can also write
+```lean
+def f (a : Array Nat) (i : Nat) (h  : i < a.size) :=
+  a[i] + a[i]
+```
+The bracket operator is whitespace sensitive.
+
+```lean
+def f (xs : List Nat) : List Nat :=
+  xs ++ xs
+
+def as : Array Nat :=
+  #[1, 2, 3, 4]
+
+def idx : Fin 4 :=
+  2
+
+#eval f [1, 2, 3] -- This is a function application
+
+#eval as[idx] -- This is an array access
+```
+The notation `a[i]` has two variants: `a[i]!` and `a[i]?`. In both cases, `i` has type `Nat`. The first one
+produces a panic error message if the index `i` is out of bounds. The latter returns an `Option` type.
+
+```lean
+#eval #['a', 'b', 'c'][1]?
+-- some 'b'
+#eval #['a', 'b', 'c'][5]?
+-- none
+#eval #['a', 'b', 'c'][1]!
+-- 'b!
+```

doc/autobound.md

@@ -0,0 +1,47 @@
+## Auto Bound Implicit Arguments
+
+In the previous section, we have shown how implicit arguments make functions more convenient to use.
+However, functions such as `compose` are still quite verbose to define. Note that the universe
+polymorphic `compose` is even more verbose than the one previously defined.
+
+```lean
+universe u v w
+def compose {α : Type u} {β : Type v} {γ : Type w}
+            (g : β → γ) (f : α → β) (x : α) : γ :=
+  g (f x)
+```
+
+You can avoid the `universe` command by providing the universe parameters when defining `compose`.
+
+```lean
+def compose.{u, v, w}
+            {α : Type u} {β : Type v} {γ : Type w}
+            (g : β → γ) (f : α → β) (x : α) : γ :=
+  g (f x)
+```
+
+Lean 4 supports a new feature called *auto bound implicit arguments*. It makes functions such as
+`compose` much more convenient to write. When Lean processes the header of a declaration,
+any unbound identifier is automatically added as an implicit argument *if* it is a single lower case or
+greek letter. With this feature, we can write `compose` as
+
+```lean
+def compose (g : β → γ) (f : α → β) (x : α) : γ :=
+  g (f x)
+
+#check @compose
+-- {β : Sort u_1} → {γ : Sort u_2} → {α : Sort u_3} → (β → γ) → (α → β) → α → γ
+```
+Note that, Lean inferred a more general type using `Sort` instead of `Type`.
+
+Although we love this feature and use it extensively when implementing Lean,
+we realize some users may feel uncomfortable with it. Thus, you can disable it using
+the command `set_option autoImplicit false`.
+```lean
+set_option autoImplicit false
+/- The following definition produces `unknown identifier` errors -/
+-- def compose (g : β → γ) (f : α → β) (x : α) : γ :=
+--   g (f x)
+```
+The Lean language server provides [semantic highlighting](./semantic_highlighting.md) information to editors, and it provides
+visual feedback whether an identifier has been interpreted as an auto bound implicit argument.

doc/book.toml

@@ -3,7 +3,7 @@ authors = ["Leonardo de Moura", "Sebastian Ullrich"]
 language = "en"
 multilingual = false
 src = "."
-title = "Lean Documentation Overview"
+title = "Lean Manual"
 
 [build]
 build-dir = "out"

doc/builtintypes.md

@@ -0,0 +1,25 @@
+# Builtin Types
+
+## Numeric Operations
+
+Lean supports the basic mathematical operations you’d expect for all of the number types: addition, subtraction, multiplication, division, and remainder.
+The following code shows how you’d use each one in a `def` commands:
+
+```lean
+-- addition
+def sum := 5 + 10
+
+-- subtraction
+def difference := 95.5 - 4.3
+
+-- multiplication
+def product := 4 * 30
+
+-- division
+def quotient := 53.7 / 32.2
+
+-- remainder/modulo
+def modulo := 43 % 5
+```
+
+Each expression in these statements uses a mathematical operator and evaluates to a single value.

doc/char.md

@@ -0,0 +1,11 @@
+# Characters
+
+A value of type `Char`, also known as a character, is a [Unicode scalar value](https://www.unicode.org/glossary/#unicode_scalar_value). It is represented using an unsigned 32-bit integer and is statically guaranteed to be a valid Unicode scalar value.
+
+Syntactically, character literals are enclosed in single quotes.
+```lean
+#eval 'a' -- 'a'
+#eval '∀' -- '∀'
+```
+
+Characters are ordered and can be decidably compared using the relational operators `=`, `<`, `≤`, `>`, `≥`.

doc/decltypes.md

@@ -0,0 +1,29 @@
+# Declaring New Types
+
+In Lean's library, every concrete type other than the universes and every type constructor other than the dependent function type is
+an instance of a general family of type constructions known as *inductive types*. It is remarkable that it is possible to develop
+complex programs and formalize mathematics based on nothing more than the type universes, dependent function types,
+and inductive types; everything else follows from those.
+
+Intuitively, an inductive type is built up from a specified list of constructors. In Lean, the basic syntax for specifying such a type is as follows:
+```
+inductive NewType where
+  | constructor_1 : ... → NewType
+  | constructor_2 : ... → NewType
+  ...
+  | constructor_n : ... → NewType
+```
+
+The intuition is that each constructor specifies a way of building new objects of ``NewType``, possibly from previously constructed values.
+The type ``NewType`` consists of nothing more than the objects that are constructed in this way.
+
+We will see below that the arguments to the constructors can include objects of type ``NewType``,
+subject to a certain "positivity" constraint, which guarantees that elements of ``NewType`` are built
+from the bottom up. Roughly speaking, each ``...`` can be any function type constructed from ``NewType``
+and previously defined types, in which ``NewType`` appears, if at all, only as the "target" of the function type.
+
+We will provide a number of examples of inductive types. We will also consider slight generalizations of the scheme above,
+to mutually defined inductive types, and so-called *inductive families*.
+
+Every inductive type comes with constructors, which show how to construct an element of the type, and elimination rules,
+which show how to "use" an element of the type in another construction.

doc/dev/release_checklist.md

@@ -5,6 +5,11 @@ See below for the checklist for release candidates.
 
 We'll use `v4.6.0` as the intended release version as a running example.
 
+- One week before the planned release, ensure that
+  (1) someone has written the release notes and
+  (2) someone has written the first draft of the release blog post.
+  If there is any material in `./releases_drafts/` on the `releases/v4.6.0` branch, then the release notes are not done.
+  (See the section "Writing the release notes".)
 - `git checkout releases/v4.6.0`
   (This branch should already exist, from the release candidates.)
 - `git pull`
@@ -81,7 +86,7 @@ We'll use `v4.6.0` as the intended release version as a running example.
       - Toolchain bump PR notes:
         - In addition to updating the `lean-toolchain` and `lakefile.lean`,
           in `.github/workflows/lean4checker.yml` update the line
-          `git checkout v4.6.0` to the appropriate tag.
+          `git checkout v4.6.0` to the appropriate tag. 
         - Push the PR branch to the main Mathlib repository rather than a fork, or CI may not work reliably
         - Create and push the tag
         - Create a new branch from the tag, push it, and open a pull request against `stable`.
@@ -134,13 +139,16 @@ We'll use `v4.7.0-rc1` as the intended release version in this example.
     git checkout -b releases/v4.7.0
     ```
 - In `RELEASES.md` replace `Development in progress` in the `v4.7.0` section with `Release notes to be written.`
-- It is essential to choose the nightly that will become the release candidate as early as possible, to avoid confusion.
+- We will rely on automatically generated release notes for release candidates,
+  and the written release notes will be used for stable versions only.
+  It is essential to choose the nightly that will become the release candidate as early as possible, to avoid confusion.
 - In `src/CMakeLists.txt`,
   - verify that you see `set(LEAN_VERSION_MINOR 7)` (for whichever `7` is appropriate); this should already have been updated when the development cycle began.
   - `set(LEAN_VERSION_IS_RELEASE 1)` (this should be a change; on `master` and nightly releases it is always `0`).
   - Commit your changes to `src/CMakeLists.txt`, and push.
 - `git tag v4.7.0-rc1`
 - `git push origin v4.7.0-rc1`
+- Ping the FRO Zulip that release notes need to be written. The release notes do not block completing the rest of this checklist.
 - Now wait, while CI runs.
   - You can monitor this at `https://github.com/leanprover/lean4/actions/workflows/ci.yml`, looking for the `v4.7.0-rc1` tag.
   - This step can take up to an hour.
@@ -240,12 +248,15 @@ Please read https://leanprover-community.github.io/contribute/tags_and_branches.
 
 # Writing the release notes
 
-Release notes are automatically generated from the commit history, using `script/release_notes.py`.
+We are currently trying a system where release notes are compiled all at once from someone looking through the commit history.
+The exact steps are a work in progress.
+Here is the general idea:
 
-Run this as `script/release_notes.py v4.6.0`, where `v4.6.0` is the *previous* release version. This will generate output
-for all commits since that tag. Note that there is output on both stderr, which should be manually reviewed,
-and on stdout, which should be manually copied to `RELEASES.md`.
-
-There can also be pre-written entries in `./releases_drafts`, which should be all incorporated in the release notes and then deleted from the branch.
+* The work is done right on the `releases/v4.6.0` branch sometime after it is created but before the stable release is made.
+  The release notes for `v4.6.0` will later be copied to `master` when we begin a new development cycle.
+* There can be material for release notes entries in commit messages.
+* There can also be pre-written entries in `./releases_drafts`, which should be all incorporated in the release notes and then deleted from the branch.
   See `./releases_drafts/README.md` for more information.
+* The release notes should be written from a downstream expert user's point of view.
 
+This section will be updated when the next release notes are written (for `v4.10.0`).

doc/do.md

@@ -0,0 +1,417 @@
+# The `do` notation
+
+Lean is a pure functional programming language, but you can write effectful code using the `do` embedded domain specific language (DSL). The following simple program prints two strings "hello" and "world" in the standard output and terminates with exit code 0. Note that the type of the program is `IO UInt32`. You can read this type as the type of values that perform input-output effects and produce a value of type `UInt32`.
+
+```lean
+def main : IO UInt32 := do
+  IO.println "hello"
+  IO.println "world"
+  return 0
+```
+The type of `IO.println` is `String → IO Unit`. That is, it is a function from `String` to `IO Unit` which indicates it may perform input-output effects and produce a value of type `Unit`. We often say that functions that may perform effects are *methods*.
+We also say a method application, such as `IO.println "hello"` is an *action*.
+Note that the examples above also demonstrates that braceless `do` blocks are whitespace sensitive.
+If you like `;`s and curly braces, you can write the example above as
+```lean
+def main : IO UInt32 := do {
+  IO.println "hello";
+  IO.println "world";
+  return 0;
+}
+```
+Semicolons can be used even when curly braces are not used. They are particularly useful when you want to "pack" more than one action in a single line.
+```lean
+def main : IO UInt32 := do
+  IO.println "hello"; IO.println "world"
+  return 0
+```
+Whitespace sensitivity in programming languages is a controversial topic
+among programmers. You should use your own style. We, the Lean developers, **love** the
+braceless and semicolon-free style.
+We believe it is clean and beautiful.
+
+The `do` DSL expands into the core Lean language. Let's inspect the different components using the commands `#print` and `#check`.
+
+```lean
+# def main : IO UInt32 := do
+#  IO.println "hello"
+#  IO.println "world"
+#  return 0
+
+#check IO.println "hello"
+-- IO Unit
+#print main
+-- Output contains the infix operator `>>=` and `pure`
+-- The following `set_option` disables notation such as `>>=` in the output
+set_option pp.notation false in
+#print main
+-- Output contains `bind` and `pure`
+#print bind
+-- bind : {m : Type u → Type v} → [self : Bind m] → {α β : Type u} →
+--        m α → (α → m β) → m β
+#print pure
+-- pure : {m : Type u → Type v} → [self : Pure m] → {α : Type u} →
+--        α → m α
+
+-- IO implements the type classes `Bind` and `Pure`.
+#check (inferInstance : Bind IO)
+#check (inferInstance : Pure IO)
+```
+The types of `bind` and `pure` may look daunting at first sight.
+They both have many implicit arguments. Let's focus first on the explicit arguments.
+`bind` has two explicit arguments `m α` and `α → m β`. The first one should
+be viewed as an action with effects `m` and producing a value of type `α`.
+The second is a function that takes a value of type `α` and produces an action
+with effects `m` and a value of type `β`. The result is `m β`. The method `bind` is composing
+these two actions. We often say `bind` is an abstract semicolon. The method `pure` converts
+a value `α` into an action that produces an action `m α`.
+
+Here is the same function being defined using `bind` and `pure` without the `do` DSL.
+```lean
+def main : IO UInt32 :=
+  bind (IO.println "hello") fun _ =>
+  bind (IO.println "world") fun _ =>
+  pure 0
+```
+
+The notations `let x <- action1; action2` and `let x ← action1; action2` are just syntax sugar for `bind action1 fun x => action2`.
+Here is a small example using it.
+```lean
+def isGreaterThan0 (x : Nat) : IO Bool := do
+  IO.println s!"value: {x}"
+  return x > 0
+
+def f (x : Nat) : IO Unit := do
+  let c <- isGreaterThan0 x
+  if c then
+    IO.println s!"{x} is greater than 0"
+  else
+    pure ()
+
+#eval f 10
+-- value: 10
+-- 10 is greater than 0
+```
+
+
+## Nested actions
+
+Note that we cannot write `if isGreaterThan0 x then ... else ...` because the condition in a `if-then-else` is a **pure** value without effects, but `isGreaterThan0 x` has type `IO Bool`. You can use the nested action notation to avoid this annoyance. Here is an equivalent definition for `f` using a nested action.
+```lean
+# def isGreaterThan0 (x : Nat) : IO Bool := do
+#  IO.println s!"x: {x}"
+#  return x > 0
+
+def f (x : Nat) : IO Unit := do
+  if (<- isGreaterThan0 x) then
+    IO.println s!"{x} is greater than 0"
+  else
+    pure ()
+
+#print f
+```
+Lean "lifts" the nested actions and introduces the `bind` for us.
+Here is an example with two nested actions. Note that both actions are executed
+even if `x = 0`.
+```lean
+# def isGreaterThan0 (x : Nat) : IO Bool := do
+#  IO.println s!"x: {x}"
+#  return x > 0
+
+def f (x y : Nat) : IO Unit := do
+  if (<- isGreaterThan0 x) && (<- isGreaterThan0 y) then
+    IO.println s!"{x} and {y} are greater than 0"
+  else
+    pure ()
+
+#eval f 0 10
+-- value: 0
+-- value: 10
+
+-- The function `f` above is equivalent to
+def g (x y : Nat) : IO Unit := do
+  let c1 <- isGreaterThan0 x
+  let c2 <- isGreaterThan0 y
+  if c1 && c2 then
+    IO.println s!"{x} and {y} are greater than 0"
+  else
+    pure ()
+
+theorem fgEqual : f = g :=
+  rfl -- proof by reflexivity
+```
+Here are two ways to achieve the short-circuit semantics in the example above
+```lean
+# def isGreaterThan0 (x : Nat) : IO Bool := do
+#  IO.println s!"x: {x}"
+#  return x > 0
+
+def f1 (x y : Nat) : IO Unit := do
+  if (<- isGreaterThan0 x <&&> isGreaterThan0 y) then
+    IO.println s!"{x} and {y} are greater than 0"
+  else
+    pure ()
+
+-- `<&&>` is the effectful version of `&&`
+-- Given `x y : IO Bool`, `x <&&> y` : m Bool`
+-- It only executes `y` if `x` returns `true`.
+
+#eval f1 0 10
+-- value: 0
+#eval f1 1 10
+-- value: 1
+-- value: 10
+-- 1 and 10 are greater than 0
+
+def f2 (x y : Nat) : IO Unit := do
+  if (<- isGreaterThan0 x) then
+    if (<- isGreaterThan0 y) then
+      IO.println s!"{x} and {y} are greater than 0"
+    else
+      pure ()
+  else
+    pure ()
+```
+
+## `if-then` notation
+
+In the `do` DSL, we can write `if c then action` as a shorthand for `if c then action else pure ()`. Here is the method `f2` using this shorthand.
+```lean
+# def isGreaterThan0 (x : Nat) : IO Bool := do
+#  IO.println s!"x: {x}"
+#  return x > 0
+
+def f2 (x y : Nat) : IO Unit := do
+  if (<- isGreaterThan0 x) then
+    if (<- isGreaterThan0 y) then
+      IO.println s!"{x} and {y} are greater than 0"
+```
+
+## Reassignments
+
+When writing effectful code, it is natural to think imperatively.
+For example, suppose we want to create an empty array `xs`,
+add `0` if some condition holds, add `1` if another condition holds,
+and then print it. In the following example, we use variable
+"shadowing" to simulate this kind of "update".
+
+```lean
+def f (b1 b2 : Bool) : IO Unit := do
+  let xs := #[]
+  let xs := if b1 then xs.push 0 else xs
+  let xs := if b2 then xs.push 1 else xs
+  IO.println xs
+
+#eval f true true
+-- #[0, 1]
+#eval f false true
+-- #[1]
+#eval f true false
+-- #[0]
+#eval f false false
+-- #[]
+```
+
+We can use tuples to simulate updates on multiple variables.
+
+```lean
+def f (b1 b2 : Bool) : IO Unit := do
+  let xs := #[]
+  let ys := #[]
+  let (xs, ys) := if b1 then (xs.push 0, ys) else (xs, ys.push 0)
+  let (xs, ys) := if b2 then (xs.push 1, ys) else (xs, ys.push 1)
+  IO.println s!"xs: {xs}, ys: {ys}"
+
+#eval f true false
+-- xs: #[0], ys: #[1]
+```
+
+We can also simulate the control-flow above using *join-points*.
+A join-point is a `let` that is always tail called and fully applied.
+The Lean compiler implements them using `goto`s.
+Here is the same example using join-points.
+
+```lean
+def f (b1 b2 : Bool) : IO Unit := do
+  let jp1 xs ys := IO.println s!"xs: {xs}, ys: {ys}"
+  let jp2 xs ys := if b2 then jp1 (xs.push 1) ys else jp1 xs (ys.push 1)
+  let xs := #[]
+  let ys := #[]
+  if b1 then jp2 (xs.push 0) ys else jp2 xs (ys.push 0)
+
+#eval f true false
+-- xs: #[0], ys: #[1]
+```
+
+You can capture complex control-flow using join-points.
+The `do` DSL offers the variable reassignment feature to make this kind of code more comfortable to write. In the following example, the `mut` modifier at `let mut xs := #[]` indicates that variable `xs` can be reassigned. The example contains two reassignments `xs := xs.push 0` and `xs := xs.push 1`. The reassignments are compiled using join-points. There is no hidden state being updated.
+
+```lean
+def f (b1 b2 : Bool) : IO Unit := do
+  let mut xs := #[]
+  if b1 then xs := xs.push 0
+  if b2 then xs := xs.push 1
+  IO.println xs
+
+#eval f true true
+-- #[0, 1]
+```
+The notation `x <- action` reassigns `x` with the value produced by the action. It is equivalent to `x := (<- action)`
+
+## Iteration
+
+The `do` DSL provides a unified notation for iterating over datastructures. Here are a few examples.
+
+```lean
+def sum (xs : Array Nat) : IO Nat := do
+  let mut s := 0
+  for x in xs do
+    IO.println s!"x: {x}"
+    s := s + x
+  return s
+
+#eval sum #[1, 2, 3]
+-- x: 1
+-- x: 2
+-- x: 3
+-- 6
+
+-- We can write pure code using the `Id.run <| do` DSL too.
+def sum' (xs : Array Nat) : Nat := Id.run <| do
+  let mut s := 0
+  for x in xs do
+    s := s + x
+  return s
+
+#eval sum' #[1, 2, 3]
+-- 6
+
+def sumEven (xs : Array Nat) : IO Nat := do
+  let mut s := 0
+  for x in xs do
+    if x % 2 == 0 then
+      IO.println s!"x: {x}"
+      s := s + x
+  return s
+
+#eval sumEven #[1, 2, 3, 6]
+-- x: 2
+-- x: 6
+-- 8
+
+def splitEvenOdd (xs : List Nat) : IO Unit := do
+  let mut evens := #[]
+  let mut odds  := #[]
+  for x in xs do
+    if x % 2 == 0 then
+      evens := evens.push x
+    else
+      odds := odds.push x
+  IO.println s!"evens: {evens}, odds: {odds}"
+
+#eval splitEvenOdd [1, 2, 3, 4]
+-- evens: #[2, 4], odds: #[1, 3]
+
+def findNatLessThan (x : Nat) (p : Nat → Bool) : IO Nat := do
+  -- [:x] is notation for the range [0, x)
+  for i in [:x] do
+    if p i then
+      return i -- `return` from the `do` block
+  throw (IO.userError "value not found")
+
+#eval findNatLessThan 10 (fun x => x > 5 && x % 4 == 0)
+-- 8
+
+def sumOddUpTo (xs : List Nat) (threshold : Nat) : IO Nat := do
+  let mut s := 0
+  for x in xs do
+    if x % 2 == 0 then
+      continue -- it behaves like the `continue` statement in imperative languages
+    IO.println s!"x: {x}"
+    s := s + x
+    if s > threshold then
+      break -- it behaves like the `break` statement in imperative languages
+  IO.println s!"result: {s}"
+  return s
+
+#eval sumOddUpTo [2, 3, 4, 11, 20, 31, 41, 51, 107] 40
+-- x: 3
+-- x: 11
+-- x: 31
+-- result: 45
+-- 45
+```
+
+TODO: describe `forIn`
+
+## Try-catch
+
+TODO
+
+## Returning early from a failed match
+
+Inside a `do` block, the pattern `let _ ← <success> | <fail>` will continue with the rest of the block if the match on the left hand side succeeds, but will execute the right hand side and exit the block on failure:
+
+```lean
+def showUserInfo (getUsername getFavoriteColor : IO (Option String)) : IO Unit := do
+  let some n ← getUsername | IO.println "no username!"
+  IO.println s!"username: {n}"
+  let some c ← getFavoriteColor | IO.println "user didn't provide a favorite color!"
+  IO.println s!"favorite color: {c}"
+
+-- username: JohnDoe
+-- favorite color: red
+#eval showUserInfo (pure <| some "JohnDoe") (pure <| some "red")
+
+-- no username
+#eval showUserInfo (pure none) (pure <| some "purple")
+
+-- username: JaneDoe
+-- user didn't provide a favorite color
+#eval showUserInfo (pure <| some "JaneDoe") (pure none)
+```
+
+## If-let
+
+Inside a `do` block, users can employ the `if let` pattern to destructure actions:
+
+```lean
+def tryIncrement (getInput : IO (Option Nat)) : IO (Except String Nat) := do
+  if let some n ← getInput
+  then return Except.ok n.succ
+  else return Except.error "argument was `none`"
+
+-- Except.ok 2
+#eval tryIncrement (pure <| some 1)
+
+-- Except.error "argument was `none`"
+#eval tryIncrement (pure <| none)
+```
+
+## Pattern matching
+
+TODO
+
+## Monads
+
+TODO
+
+## ReaderT
+
+TODO
+
+## StateT
+
+TODO
+
+## StateRefT
+
+TODO
+
+## ExceptT
+
+TODO
+
+## MonadLift and automatic lifting
+
+TODO

doc/elaborators.md

@@ -0,0 +1,8 @@
+## Elaborators
+
+TODO. See [Lean Together 2021: Metaprogramming in Lean
+4](https://youtu.be/hxQ1vvhYN_U) for an overview as well [the
+continuation](https://youtu.be/vy4JWIiiXSY) about tactic programming.
+For more information on antiquotations, see also §4.1 of [Beyond
+Notations: Hygienic Macro Expansion for Theorem Proving
+Languages](https://arxiv.org/pdf/2001.10490.pdf#page=11).

doc/enum.md

@@ -0,0 +1,190 @@
+# Enumerated Types
+
+The simplest kind of inductive type is simply a type with a finite, enumerated list of elements.
+The following command declares the enumerated type `Weekday`.
+```lean
+inductive Weekday where
+  | sunday    : Weekday
+  | monday    : Weekday
+  | tuesday   : Weekday
+  | wednesday : Weekday
+  | thursday  : Weekday
+  | friday    : Weekday
+  | saturday  : Weekday
+```
+
+The `Weekday` type has 7 constructors/elements. The constructors live in the `Weekday` namespace
+Think of `sunday`, `monday`, …, `saturday` as being distinct elements of `Weekday`,
+with no other distinguishing properties.
+```lean
+# inductive Weekday where
+#  | sunday    : Weekday
+#  | monday    : Weekday
+#  | tuesday   : Weekday
+#  | wednesday : Weekday
+#  | thursday  : Weekday
+#  | friday    : Weekday
+#  | saturday  : Weekday
+#check Weekday.sunday   -- Weekday
+#check Weekday.monday   -- Weekday
+```
+
+You can define functions by pattern matching.
+The following function converts a `Weekday` into a natural number.
+```lean
+# inductive Weekday where
+#  | sunday    : Weekday
+#  | monday    : Weekday
+#  | tuesday   : Weekday
+#  | wednesday : Weekday
+#  | thursday  : Weekday
+#  | friday    : Weekday
+#  | saturday  : Weekday
+def natOfWeekday (d : Weekday) : Nat :=
+  match d with
+  | Weekday.sunday    => 1
+  | Weekday.monday    => 2
+  | Weekday.tuesday   => 3
+  | Weekday.wednesday => 4
+  | Weekday.thursday  => 5
+  | Weekday.friday    => 6
+  | Weekday.saturday  => 7
+
+#eval natOfWeekday Weekday.tuesday -- 3
+```
+
+It is often useful to group definitions related to a type in a namespace with the same name.
+For example, we can put the function above into the ``Weekday`` namespace.
+We are then allowed to use the shorter name when we open the namespace.
+
+In the following example, we define functions from ``Weekday`` to ``Weekday`` in the namespace `Weekday`.
+```lean
+# inductive Weekday where
+#  | sunday    : Weekday
+#  | monday    : Weekday
+#  | tuesday   : Weekday
+#  | wednesday : Weekday
+#  | thursday  : Weekday
+#  | friday    : Weekday
+#  | saturday  : Weekday
+namespace Weekday
+
+def next (d : Weekday) : Weekday :=
+  match d with
+  | sunday    => monday
+  | monday    => tuesday
+  | tuesday   => wednesday
+  | wednesday => thursday
+  | thursday  => friday
+  | friday    => saturday
+  | saturday  => sunday
+
+end Weekday
+```
+It is so common to start a definition with a `match` in Lean, that Lean provides a syntax sugar for it.
+```lean
+# inductive Weekday where
+#  | sunday    : Weekday
+#  | monday    : Weekday
+#  | tuesday   : Weekday
+#  | wednesday : Weekday
+#  | thursday  : Weekday
+#  | friday    : Weekday
+#  | saturday  : Weekday
+# namespace Weekday
+def previous : Weekday -> Weekday
+  | sunday    => saturday
+  | monday    => sunday
+  | tuesday   => monday
+  | wednesday => tuesday
+  | thursday  => wednesday
+  | friday    => thursday
+  | saturday  => friday
+# end Weekday
+```
+We can use the command `#eval` to test our definitions.
+```lean
+# inductive Weekday where
+#  | sunday    : Weekday
+#  | monday    : Weekday
+#  | tuesday   : Weekday
+#  | wednesday : Weekday
+#  | thursday  : Weekday
+#  | friday    : Weekday
+#  | saturday  : Weekday
+# namespace Weekday
+# def next (d : Weekday) : Weekday :=
+#  match d with
+#  | sunday    => monday
+#  | monday    => tuesday
+#  | tuesday   => wednesday
+#  | wednesday => thursday
+#  | thursday  => friday
+#  | friday    => saturday
+#  | saturday  => sunday
+# def previous : Weekday -> Weekday
+#  | sunday    => saturday
+#  | monday    => sunday
+#  | tuesday   => monday
+#  | wednesday => tuesday
+#  | thursday  => wednesday
+#  | friday    => thursday
+#  | saturday  => friday
+def toString : Weekday -> String
+  | sunday    => "Sunday"
+  | monday    => "Monday"
+  | tuesday   => "Tuesday"
+  | wednesday => "Wednesday"
+  | thursday  => "Thursday"
+  | friday    => "Friday"
+  | saturday  => "Saturday"
+
+#eval toString (next sunday)             -- "Monday"
+#eval toString (next tuesday)            -- "Wednesday"
+#eval toString (previous wednesday)      -- "Tuesday"
+#eval toString (next (previous sunday))  -- "Sunday"
+#eval toString (next (previous monday))  -- "Monday"
+-- ..
+# end Weekday
+```
+We can now prove the general theorem that ``next (previous d) = d`` for any weekday ``d``.
+The idea is to perform a proof by cases using `match`, and rely on the fact for each constructor both
+sides of the equality reduce to the same term.
+```lean
+# inductive Weekday where
+#  | sunday    : Weekday
+#  | monday    : Weekday
+#  | tuesday   : Weekday
+#  | wednesday : Weekday
+#  | thursday  : Weekday
+#  | friday    : Weekday
+#  | saturday  : Weekday
+# namespace Weekday
+# def next (d : Weekday) : Weekday :=
+#  match d with
+#  | sunday    => monday
+#  | monday    => tuesday
+#  | tuesday   => wednesday
+#  | wednesday => thursday
+#  | thursday  => friday
+#  | friday    => saturday
+#  | saturday  => sunday
+# def previous : Weekday -> Weekday
+#  | sunday    => saturday
+#  | monday    => sunday
+#  | tuesday   => monday
+#  | wednesday => tuesday
+#  | thursday  => wednesday
+#  | friday    => thursday
+#  | saturday  => friday
+theorem nextOfPrevious (d : Weekday) : next (previous d) = d :=
+  match d with
+  | sunday    => rfl
+  | monday    => rfl
+  | tuesday   => rfl
+  | wednesday => rfl
+  | thursday  => rfl
+  | friday    => rfl
+  | saturday  => rfl
+# end Weekday
+```

doc/flake.nix

@@ -83,6 +83,7 @@
         src = ./.;
         roots = [
           { mod = "examples"; glob = "submodules"; }
+          { mod = "monads"; glob = "submodules"; }
         ];
       };
       inked = renderPackage literate;

doc/float.md

@@ -0,0 +1 @@
+# Float

doc/functions.md

@@ -0,0 +1,153 @@
+# Functions
+
+Functions are the fundamental unit of program execution in any programming language.
+As in other languages, a Lean function has a name, can have parameters and take arguments, and has a body.
+Lean also supports functional programming constructs such as treating functions as values,
+using unnamed functions in expressions, composition of functions to form new functions,
+curried functions, and the implicit definition of functions by way of
+the partial application of function arguments.
+
+You define functions by using the `def` keyword followed by its name, a parameter list, return type and its body.
+The parameter list consists of successive parameters that are separated by spaces.
+You can specify an explicit type for each parameter.
+If you do not specify a specific argument type, the compiler tries to infer the type from the function body.
+An error is returned when it cannot be inferred.
+The expression that makes up the function body is typically a compound expression consisting of a number of expressions
+that culminate in a final expression that is the return value.
+The return type is a colon followed by a type and is optional.
+If you do not specify the type of the return value explicitly,
+the compiler tries to determine the return type from the final expression.
+
+```lean
+def f x := x + 1
+```
+In the previous example, the function name is `f`, the argument is `x`, which has type `Nat`,
+the function body is `x + 1`, and the return value is of type `Nat`.
+The following example defines the factorial recursive function using pattern matching.
+```lean
+def fact x :=
+  match x with
+  | 0   => 1
+  | n+1 => (n+1) * fact n
+
+#eval fact 100
+```
+By default, Lean only accepts total functions.
+The `partial` keyword may be used to define a recursive function without a termination proof; `partial` functions compute in compiled programs, but are opaque in proofs and during type checking.
+```lean
+partial def g (x : Nat) (p : Nat -> Bool) : Nat :=
+  if p x then
+    x
+  else
+    g (x+1) p
+
+#eval g 0 (fun x => x > 10)
+```
+In the previous example, `g x p` only terminates if there is a `y >= x` such that `p y` returns `true`.
+Of course, `g 0 (fun x => false)` never terminates.
+
+However, the use of `partial` is restricted to functions whose return type is not empty so the soundness
+of the system is not compromised.
+
+```lean,ignore
+partial def loop? : α := -- failed to compile partial definition 'loop?', failed to
+  loop?                  -- show that type is inhabited and non empty
+
+partial def loop [Inhabited α] : α := -- compiles
+  loop
+
+example : True := -- accepted
+  loop
+
+example : False :=
+  loop -- failed to synthesize instance Inhabited False
+```
+
+If we were able to partially define `loop?`, we could prove `False` with it.
+
+# Lambda expressions
+
+A lambda expression is an unnamed function.
+You define lambda expressions by using the `fun` keyword. A lambda expression resembles a function definition, except that instead of the `:=` token,
+the `=>` token is used to separate the argument list from the function body. As in a regular function definition,
+the argument types can be inferred or specified explicitly, and the return type of the lambda expression is inferred from the type of the
+last expression in the body.
+
+```lean
+def twice (f : Nat -> Nat) (x : Nat) : Nat :=
+  f (f x)
+
+#eval twice (fun x => x + 1) 3
+#eval twice (fun (x : Nat) => x * 2) 3
+
+#eval List.map (fun x => x + 1) [1, 2, 3]
+-- [2, 3, 4]
+
+#eval List.map (fun (x, y) => x + y) [(1, 2), (3, 4)]
+-- [3, 7]
+```
+
+# Syntax sugar for simple lambda expressions
+
+Simple functions can be defined using parentheses and `·` as a placeholder.
+```lean
+#check (· + 1)
+-- fun a => a + 1
+#check (2 - ·)
+-- fun a => 2 - a
+#eval [1, 2, 3, 4, 5].foldl (· * ·) 1
+-- 120
+
+def h (x y z : Nat) :=
+  x + y + z
+
+#check (h · 1 ·)
+-- fun a b => h a 1 b
+
+#eval [(1, 2), (3, 4), (5, 6)].map (·.1)
+-- [1, 3, 5]
+```
+In the previous example, the term `(·.1)` is syntax sugar for `fun x => x.1`.
+
+# Pipelining
+
+Pipelining enables function calls to be chained together as successive operations. Pipelining works as follows:
+
+```lean
+def add1 x := x + 1
+def times2 x := x * 2
+
+#eval times2 (add1 100)
+#eval 100 |> add1 |> times2
+#eval times2 <| add1 <| 100
+```
+The result of the previous `#eval` commands is 202.
+The forward pipeline `|>` operator takes a function and an argument and return a value.
+In contrast, the backward pipeline `<|` operator takes an argument and a function and returns a value.
+These operators are useful for minimizing the number of parentheses.
+```lean
+def add1Times3FilterEven (xs : List Nat) :=
+  List.filter (· % 2 == 0) (List.map (· * 3) (List.map (· + 1) xs))
+
+#eval add1Times3FilterEven [1, 2, 3, 4]
+-- [6, 12]
+
+-- Define the same function using pipes
+def add1Times3FilterEven' (xs : List Nat) :=
+  xs |> List.map (· + 1) |> List.map (· * 3) |> List.filter (· % 2 == 0)
+
+#eval add1Times3FilterEven' [1, 2, 3, 4]
+-- [6, 12]
+```
+Lean also supports the operator `|>.` which combines forward pipeline `|>` operator with the `.` field notation.
+```lean
+-- Define the same function using pipes
+def add1Times3FilterEven'' (xs : List Nat) :=
+  xs.map (· + 1) |>.map (· * 3) |>.filter (· % 2 == 0)
+
+#eval add1Times3FilterEven'' [1, 2, 3, 4]
+-- [6, 12]
+```
+
+For users familiar with the Haskell programming language,
+Lean also supports the notation `f $ a` for the backward pipeline `f <| a`.

doc/implicit.md

@@ -0,0 +1,142 @@
+## Implicit Arguments
+
+Suppose we define the `compose` function as.
+
+```lean
+def compose (α β γ : Type) (g : β → γ) (f : α → β) (x : α) : γ :=
+  g (f x)
+```
+
+The function `compose` takes three types, ``α``, ``β``, and ``γ``, and two functions, ``g : β → γ`` and ``f : α → β``, a value `x : α`, and
+returns ``g (f x)``, the composition of ``g`` and ``f``.
+We say `compose` is polymorphic over types ``α``, ``β``, and ``γ``. Now, let's use `compose`:
+
+```lean
+# def compose (α β γ : Type) (g : β → γ) (f : α → β) (x : α) : γ :=
+#   g (f x)
+def double (x : Nat) := 2*x
+def triple (x : Nat) := 3*x
+
+#check compose Nat Nat Nat double triple 10 -- Nat
+#eval  compose Nat Nat Nat double triple 10 -- 60
+
+def appendWorld (s : String) := s ++ "world"
+#check String.length -- String → Nat
+
+#check compose String String Nat String.length appendWorld "hello" -- Nat
+#eval  compose String String Nat String.length appendWorld "hello" -- 10
+```
+
+Because `compose` is polymorphic over types ``α``, ``β``, and ``γ``, we have to provide them in the examples above.
+But this information is redundant: one can infer the types from the arguments ``g`` and ``f``.
+This is a central feature of dependent type theory: terms carry a lot of information, and often some of that information can be inferred from the context.
+In Lean, one uses an underscore, ``_``, to specify that the system should fill in the information automatically.
+
+```lean
+# def compose (α β γ : Type) (g : β → γ) (f : α → β) (x : α) : γ :=
+#  g (f x)
+# def double (x : Nat) := 2*x
+# def triple (x : Nat) := 3*x
+#check compose _ _ _ double triple 10 -- Nat
+#eval  compose Nat Nat Nat double triple 10 -- 60
+# def appendWorld (s : String) := s ++ "world"
+# #check String.length -- String → Nat
+#check compose _ _ _ String.length appendWorld "hello" -- Nat
+#eval  compose _ _ _ String.length appendWorld "hello" -- 10
+```
+It is still tedious, however, to type all these underscores. When a function takes an argument that can generally be inferred from context,
+Lean allows us to specify that this argument should, by default, be left implicit. This is done by putting the arguments in curly braces, as follows:
+
+```lean
+def compose {α β γ : Type} (g : β → γ) (f : α → β) (x : α) : γ :=
+  g (f x)
+# def double (x : Nat) := 2*x
+# def triple (x : Nat) := 3*x
+#check compose double triple 10 -- Nat
+#eval  compose double triple 10 -- 60
+# def appendWorld (s : String) := s ++ "world"
+# #check String.length -- String → Nat
+#check compose String.length appendWorld "hello" -- Nat
+#eval  compose String.length appendWorld "hello" -- 10
+```
+All that has changed are the braces around ``α β γ: Type``.
+It makes these three arguments implicit. Notationally, this hides the specification of the type,
+making it look as though ``compose`` simply takes 3 arguments.
+
+Variables can also be specified as implicit when they are declared with
+the ``variable`` command:
+
+```lean
+universe u
+
+section
+  variable {α : Type u}
+  variable (x : α)
+  def ident := x
+end
+
+variable (α β : Type u)
+variable (a : α) (b : β)
+
+#check ident
+#check ident a
+#check ident b
+```
+
+This definition of ``ident`` here has the same effect as the one above.
+
+Lean has very complex mechanisms for instantiating implicit arguments, and we will see that they can be used to infer function types, predicates, and even proofs.
+The process of instantiating these "holes," or "placeholders," in a term is part of a bigger process called *elaboration*.
+The presence of implicit arguments means that at times there may be insufficient information to fix the meaning of an expression precisely.
+An expression like ``ident`` is said to be *polymorphic*, because it can take on different meanings in different contexts.
+
+One can always specify the type ``T`` of an expression ``e`` by writing ``(e : T)``.
+This instructs Lean's elaborator to use the value ``T`` as the type of ``e`` when trying to elaborate it.
+In the following example, this mechanism is used to specify the desired types of the expressions ``ident``.
+
+```lean
+def ident {α : Type u} (a : α) : α := a
+
+#check (ident : Nat → Nat) -- Nat → Nat
+```
+
+Numerals are overloaded in Lean, but when the type of a numeral cannot be inferred, Lean assumes, by default, that it is a natural number.
+So the expressions in the first two ``#check`` commands below are elaborated in the same way, whereas the third ``#check`` command interprets ``2`` as an integer.
+
+```lean
+#check 2         -- Nat
+#check (2 : Nat) -- Nat
+#check (2 : Int) -- Int
+```
+
+Sometimes, however, we may find ourselves in a situation where we have declared an argument to a function to be implicit,
+but now want to provide the argument explicitly. If ``foo`` is such a function, the notation ``@foo`` denotes the same function with all
+the arguments made explicit.
+
+```lean
+# def ident {α : Type u} (a : α) : α := a
+variable (α β : Type)
+
+#check @ident           -- {α : Type u} → α → α
+#check @ident α         -- α → α
+#check @ident β         -- β → β
+#check @ident Nat       -- Nat → Nat
+#check @ident Bool true -- Bool
+```
+
+Notice that now the first ``#check`` command gives the type of the identifier, ``ident``, without inserting any placeholders.
+Moreover, the output indicates that the first argument is implicit.
+
+Named arguments enable you to specify an argument for a parameter by matching the argument with
+its name rather than with its position in the parameter list. You can use them to specify explicit *and* implicit arguments.
+If you don't remember the order of the parameters but know their names, you can send the arguments in any order.
+You may also provide the value for an implicit parameter when
+Lean failed to infer it. Named arguments also improve the readability of your code by identifying what
+each argument represents.
+
+```lean
+# def ident {α : Type u} (a : α) : α := a
+
+#check ident (α := Nat)  -- Nat → Nat
+#check ident (α := Bool) -- Bool → Bool
+```

doc/inductive.md

@@ -0,0 +1,3 @@
+# Inductive Types
+
+[Theorem Proving in Lean](https://lean-lang.org/theorem_proving_in_lean4/inductive_types.html) has a chapter about inductive datatypes.

doc/int.md

@@ -0,0 +1,37 @@
+# Integers
+
+The `Int` type represents the arbitrary-precision integers. There are no overflows.
+```lean
+#eval (100000000000000000 : Int) * 200000000000000000000 * 1000000000000000000000
+```
+Recall that nonnegative numerals are considered to be a `Nat` if there are no typing constraints.
+```lean
+#check 1 -- Nat
+#check -1 -- Int
+#check (1:Int) -- Int
+```
+
+The operator `/` for `Int` implements integer division.
+```lean
+#eval -10 / 4 -- -3
+```
+
+Similar to `Nat`, the internal representation of `Int` is optimized. Small integers are
+represented by a single machine word. Big integers are implemented using [GMP](https://gmplib.org/manual/) numbers.
+We recommend you use fixed precision numeric types only in performance critical code.
+
+The Lean kernel does not have special support for reducing `Int` during type checking.
+However, since `Int` is defined as
+```lean
+# namespace hidden
+inductive Int : Type where
+  | ofNat   : Nat → Int
+  | negSucc : Nat → Int
+# end hidden
+```
+the type checker will be able reduce `Int` expressions efficiently by relying on the special support for `Nat`.
+
+```lean
+theorem ex : -2000000000 * 1000000000 = -2000000000000000000 :=
+ rfl
+```

doc/list.md

@@ -0,0 +1 @@
+# List

doc/macro_overview.md

@@ -0,0 +1,393 @@
+
+# Macro Overview
+
+The official paper describing the mechanics behind Lean 4's macro system can be
+found in [Beyond Notations: Hygienic Macro Expansion for Theorem Proving
+Languages](https://arxiv.org/abs/2001.10490) by Sebastian Ullrich and Leonardo
+de Moura, and the accompanying repo with example code can be found in the
+paper's code [supplement](https://github.com/Kha/macro-supplement). The
+supplement also includes a working implementation of the macro expander, so it's
+a good case study for people interested in the details.
+
+## What is a macro in Lean?
+
+A macro is a function that takes in a syntax tree and produces a new syntax
+tree. Macros are useful for many reasons, but two of the big ones are a)
+allowing users to extend the language with new syntactic constructs without
+having to actually expand the core language, and b) allowing users to automate
+tasks that would otherwise be extremely repetitive, time-consuming, and/or
+error-prone.
+
+A motivating example is set builder notation. We would like to be able to write
+the set of natural numbers 0, 1, and 2 as just `{0, 1, 2}`. However, Lean does
+not natively support this syntax, and the actual definition of a set in Mathlib
+does not let us just declare sets in this manner; naively using the set API
+would force us to write `Set.insert 1 (Set.insert 2 (Set.singleton 3))`.
+Instead, we can teach Lean's macro system to recognize `{0, 1, 2}` as a
+shorthand for a composition of existing methods and let it do the repetitive
+work of creating the `Set.insert...` invocation for us. In this way, we can have
+our more readable and more convenient syntax without having to extend Lean
+itself, and while retaining the simple insert/singleton API.
+
+## How macros are handled
+
+The general procedure is as follows:
+
+1. Lean parses a command, creating a Lean syntax tree which contains any
+   unexpanded macros.
+
+2. Lean repeats the cycle (elaboration ~> (macro hygiene and expansion) ~>
+   elaboration...)
+
+The cycle in step 2 repeats until there are no more macros which need to be
+expanded, and elaboration can finish normally. This repetition is required since
+macros can expand to other macros, and may expand to code that needs information
+from the elaborator. As you can see, the process of macro parsing and expansion
+is interleaved with the parsing and elaboration of non-macro code.
+
+By default, macros in Lean are hygienic, which means the system avoids
+accidental name capture when reusing the same name inside and outside the macro.
+Users may occasionally want to disable hygiene, which can be accomplished with
+the command `set_option hygiene false`. More in-depth information about hygiene
+and how it's implemented in the official paper and supplement linked at the top
+of this guide.
+
+## Elements of "a" macro (important types)
+
+
+In the big picture, a macro has two components that must be implemented by the
+user, parsers and syntax transformers, where the latter is a function that says
+what the input syntax should expand to. There is a third component, syntax
+categories, such as `term`, `tactic`, and `command`, but declaring a new syntax
+category is not always necessary. When we say "parser" in the context of a
+macro, we refer to the core type `Lean.ParserDescr`, which parses elements of
+type `Lean.Syntax`, where `Lean.Syntax` represents elements of a Lean syntax
+tree. Syntax transformers are functions of type `Syntax -> MacroM Syntax`. Lean
+has a synonym for this type, which is simply `Macro`. `MacroM` is a monad that
+carries state needed for macro expansion to work nicely, including the info
+needed to implement hygiene.
+
+As an example, we again refer to Mathlib's set builder notation:
+```lean
+/- Declares a parser -/
+syntax (priority := high) "{" term,+ "}" : term
+
+/- Declares two expansions/syntax transformers -/
+macro_rules
+  | `({$x}) => `(Set.singleton $x)
+  | `({$x, $xs:term,*}) => `(Set.insert $x {$xs,*})
+
+/- Provided `Set` has been imported (from Mathlib4), these are all we need for `{1, 2, 3}` to be valid notation to create a literal set -/
+
+```
+
+This example should also make clear the reason why macros (and pretty much all
+of Lean 4's metaprogramming facilities) are functions that take an argument of
+type `Syntax` e.g. `Syntax -> MacroM Syntax`; the leading syntax element is the
+thing that actually triggers the macro expansion by matching with the declared
+parser, and as a user, you will almost always be interested in inspecting and
+transforming that initial syntax element (though there are cases in which it can
+just be ignored, as in the parameter-less exfalso tactic).
+
+Returning briefly to the API provided by Lean, `Lean.Syntax`, is pretty much
+what you would expect a basic syntax tree type to look like. Below is a slightly
+simplified representation which omits details in the `atom` and `ident`
+constructors; users can create atoms and idents which comport with this
+simplified representation using the `mkAtom` and `mkIdent` methods provided in
+the `Lean` namespace.
+```lean
+# open Lean
+inductive Syntax where
+  | missing : Syntax
+  | node (kind : SyntaxNodeKind) (args : Array Syntax) : Syntax
+  | atom : String -> Syntax
+  | ident : Name -> Syntax
+```
+
+
+
+For those interested, `MacroM` is a `ReaderT`:
+```lean
+# open Lean
+abbrev MacroM := ReaderT Macro.Context (EStateM Macro.Exception Macro.State)
+```
+
+The other relevant components are defined as follows:
+```lean
+# open Lean
+structure Context where
+  methods        : MethodsRef
+  mainModule     : Name
+  currMacroScope : MacroScope
+  currRecDepth   : Nat := 0
+  maxRecDepth    : Nat := defaultMaxRecDepth
+  ref            : Syntax
+
+inductive Exception where
+  | error             : Syntax → String → Exception
+  | unsupportedSyntax : Exception
+
+structure State where
+  macroScope : MacroScope
+  traceMsgs  : List (Prod Name String) := List.nil
+  deriving Inhabited
+```
+
+As a review/checklist, the three (sometimes only two depending on whether you
+need a new syntax category) components users need to be concerned with are:
+
+0. You may or may not need to declare a new syntax category using
+   `declare_syntax_cat`
+1. Declare a parser with either `syntax` or `macro`
+2. Declare an expansion/syntax transformer with either `macro_rules` or `macro`
+
+Parsers and syntax transformers can be declared manually, but use of the pattern
+language and `syntax`, `macro_rules`, and `macro` is recommended.
+
+## syntax categories with declare_syntax_cat
+
+`declare_syntax_cat` declares a new syntax category, like `command`, `tactic`,
+or mathlib4's `binderterm`. These are the different categories of things that
+can be referred to in a quote/antiquote. `declare_syntax_cat` results in a call
+to `registerParserCategory` and produces a new parser descriptor:
+
+```lean
+set_option trace.Elab.definition true in
+declare_syntax_cat binderterm
+
+/-
+Output:
+
+[Elab.definition.body] binderterm.quot : Lean.ParserDescr :=
+Lean.ParserDescr.node `Lean.Parser.Term.quot 1024
+  (Lean.ParserDescr.binary `andthen (Lean.ParserDescr.symbol "`(binderterm|")
+    (Lean.ParserDescr.binary `andthen (Lean.ParserDescr.cat `binderterm 0)
+      (Lean.ParserDescr.symbol ")")))
+-/
+```
+
+Declaring a new syntax category like this one automatically declares a quotation
+operator `` `(binderterm| ...)``. These pipe prefixes `<thing>|` are used in
+syntax quotations to say what category a given quotation is expected to be an
+element of. The pipe prefixes are *not* used for elements in the `term` and
+`command` categories (since they're considered the default), but need to be used
+for everything else.
+
+## Parsers and the `syntax` keyword
+
+Internally, elements of type `Lean.ParserDescr` are implemented as parser
+combinators. However, Lean offers the ability to write parsers using the
+macro/pattern language by way of the `syntax` keyword. This is the recommended
+means of writing parsers. As an example, the parser for the `rwa` (rewrite, then
+use assumption) tactic is:
+
+```lean
+# open Lean.Parser.Tactic
+set_option trace.Elab.definition true in
+syntax "rwa " rwRuleSeq (location)? : tactic
+
+/-
+which expands to:
+[Elab.definition.body] tacticRwa__ : Lean.ParserDescr :=
+Lean.ParserDescr.node `tacticRwa__ 1022
+  (Lean.ParserDescr.binary `andthen
+    (Lean.ParserDescr.binary `andthen (Lean.ParserDescr.nonReservedSymbol "rwa " false) Lean.Parser.Tactic.rwRuleSeq)
+    (Lean.ParserDescr.unary `optional Lean.Parser.Tactic.location))
+
+-/
+
+```
+
+Literals are written as double-quoted strings (`"rwa "` expects the literal
+sequence of characters `rwa`, while the trailing space provides a hint to the
+formatter that it should add a space after `rwa` when pretty printing this
+syntax); `rwRuleSeq` and `location` are themselves `ParserDescr`s, and we finish
+with `: tactic` specifying that the preceding parser is for an element in the
+`tactic` syntax category. The parentheses around `(location)?` are necessary
+(rather than `location?`) because Lean 4 allows question marks to be used in
+identifiers, so `location?` is one single identifier that ends with a question
+mark, which is not what we want.
+
+The name `tacticRwa__` is automatically generated. You can name parser
+descriptors declared with the `syntax` keyword like so:
+
+```lean
+set_option trace.Elab.definition true in
+syntax (name := introv) "introv " (colGt ident)* : tactic
+
+/-
+[Elab.definition.body] introv : Lean.ParserDescr :=
+Lean.ParserDescr.node `introv 1022
+  (Lean.ParserDescr.binary `andthen (Lean.ParserDescr.nonReservedSymbol "introv " false)
+    (Lean.ParserDescr.unary `many
+      (Lean.ParserDescr.binary `andthen (Lean.ParserDescr.const `colGt) (Lean.ParserDescr.const `ident))))
+-/
+```
+
+## The pattern language
+
+Available quantifiers are `?` (one or zero occurrences, see note below), `*`
+(zero or more occurrences), and `+` (one or more occurrences).
+
+Keep in mind that Lean makes `?` available for use in identifiers, so if we want
+a parser to look for an optional `location`, we would need to write
+`(location)?` with parenthesis acting as a separator, since `location?` would
+look for something under the identifier `location?` (where the `?` is part of
+the identifier).
+
+Parentheses can be used as delimiters.
+
+Separated lists can be constructed like so: `$ts,*` for a comma separated list.
+
+"extended splices" can be constructed as `$[..]`. See the official paper (p. 12)
+for more details.
+
+Literals are written as double-quoted strings. A literal may use trailing
+whitespace (see e.g. the `rwa` or `introv` tactics) to tell the pretty-printer
+how it should be displayed, but such whitespace will not prevent a literal with
+no trailing whitespace from matching. The spaces are relevant, but not
+interpreted literally. When the ParserDescr is turned into a Parser, the actual
+token matcher [uses the .trim of the provided
+string](https://github.com/leanprover/lean4/blob/53ec43ff9b8f55989b12c271e368287b7b997b54/src/Lean/Parser/Basic.lean#L1193),
+but the generated formatter [uses the spaces as
+specified](https://github.com/leanprover/lean4/blob/8d370f151f7c88a687152a5b161dcb484c446ce2/src/Lean/PrettyPrinter/Formatter.lean#L328),
+that is, turning the atom "rwa" in the syntax into the string rwa as part of the
+pretty printed output.
+
+## Syntax expansions with `macro_rules`, and how it desugars.
+
+`macro_rules` lets you declare expansions for a given `Syntax` element using a
+syntax similar to a `match` statement. The left-hand side of a match arm is a
+quotation (with a leading `<cat>|` for categories other than `term` and
+`command`) in which users can specify the pattern they'd like to write an
+expansion for. The right-hand side returns a syntax quotation which is the
+output the user wants to expand to.
+
+A feature of Lean's macro system is that if there are multiple expansions for a
+particular match, Lean will try the most recently declared expansion first, and
+will retry with other matching expansions if the previous attempt failed. This
+is particularly useful for extending existing tactics.
+
+The following example shows both the retry behavior, and the fact that macros
+declared using the shorthand `macro` syntax can still have additional expansions
+declared with `macro_rules`. This `transitivity` tactic is implemented such that
+it will work for either Nat.le or Nat.lt. The Nat.lt version was declared "most
+recently", so it will be tried first, but if it fails (for example, if the
+actual term in question is Nat.le) the next potential expansion will be tried:
+```lean
+macro "transitivity" e:(colGt term) : tactic => `(tactic| apply Nat.le_trans (m := $e))
+macro_rules
+  | `(tactic| transitivity $e) => `(tactic| apply Nat.lt_trans (m := $e))
+
+example (a b c : Nat) (h0 : a < b) (h1 : b < c) : a < c := by
+  transitivity b <;>
+  assumption
+
+example (a b c : Nat) (h0 : a <= b) (h1 : b <= c) : a <= c := by
+  transitivity b <;>
+  assumption
+
+/- This will fail, but is interesting in that it exposes the "most-recent first" behavior, since the
+  error message complains about being unable to unify mvar1 <= mvar2, rather than mvar1 < mvar2. -/
+/-
+example (a b c : Nat) (h0 : a <= b) (h1 : b <= c) : False := by
+  transitivity b <;>
+  assumption
+-/
+```
+
+To see the desugared definition of the actual expansion, we can again use
+`set_option trace.Elab.definition true in` and observe the output of the humble
+`exfalso` tactic defined in Mathlib4:
+```lean
+
+set_option trace.Elab.definition true in
+macro "exfalso" : tactic => `(tactic| apply False.elim)
+
+/-
+Results in the expansion:
+
+[Elab.definition.body] _aux___macroRules_tacticExfalso_1 : Lean.Macro :=
+fun x =>
+  let discr := x;
+  /- This is where Lean tries to actually identify that it's an invocation of the exfalso tactic -/
+  if Lean.Syntax.isOfKind discr `tacticExfalso = true then
+    let discr := Lean.Syntax.getArg discr 0;
+    let x := discr;
+    do
+      /- Lean getting scope/meta info from the macro monad -/
+      let info ← Lean.MonadRef.mkInfoFromRefPos
+      let scp ← Lean.getCurrMacroScope
+      let mainModule ← Lean.getMainModule
+      pure
+          (Lean.Syntax.node Lean.SourceInfo.none `Lean.Parser.Tactic.seq1
+            #[Lean.Syntax.node Lean.SourceInfo.none `null
+                #[Lean.Syntax.node Lean.SourceInfo.none `Lean.Parser.Tactic.apply
+                    #[Lean.Syntax.atom info "apply",
+                      Lean.Syntax.ident info (String.toSubstring "False.elim")
+                        (Lean.addMacroScope mainModule `False.elim scp) [(`False.elim, [])]]]])
+  else
+    /- If this wasn't actually an invocation of the exfalso tactic, throw the "unsupportedSyntax" error -/
+    let discr := x;
+    throw Lean.Macro.Exception.unsupportedSyntax
+-/
+```
+
+We can also create the syntax transformer declaration ourselves instead of using
+`macro_rules`. We'll need to name our parser and use the attribute `@[macro
+myExFalsoParser]` to associate our declaration with the parser:
+```lean
+# open Lean
+syntax (name := myExfalsoParser) "myExfalso" : tactic
+
+-- remember that `Macro` is a synonym for `Syntax -> TacticM Unit`
+@[macro myExfalsoParser] def implMyExfalso : Macro :=
+fun stx => `(tactic| apply False.elim)
+
+example (p : Prop) (h : p) (f : p -> False) : 3 = 2 := by
+  myExfalso
+  exact f h
+```
+
+In the above example, we're still using the sugar Lean provides for creating
+quotations, as it feels more intuitive and saves us some work. It is possible to
+forego the sugar altogether:
+```lean
+syntax (name := myExfalsoParser) "myExfalso" : tactic
+
+@[macro myExfalsoParser] def implMyExfalso : Lean.Macro :=
+  fun stx => pure (Lean.mkNode `Lean.Parser.Tactic.apply
+    #[Lean.mkAtomFrom stx "apply", Lean.mkCIdentFrom stx ``False.elim])
+
+example (p : Prop) (h : p) (f : p -> False) : 3 = 2 := by
+  myExfalso
+  exact f h
+```
+
+## The `macro` keyword
+
+`macro` is a shortcut which allows users to declare both a parser and an
+expansion at the same time as a matter of convenience. Additional expansions for
+the parser generated by the `macro` invocation can be added with a separate
+`macro_rules` block (see the example in the `macro_rules` section).
+
+## Unexpanders
+
+TODO; for now, see the unexpander in Mathlib.Set for an example.
+
+## More illustrative examples:
+
+The
+[Tactic.Basic](https://github.com/leanprover-community/mathlib4/blob/master/Mathlib/Tactic/Basic.lean)
+file in Mathlib4 contains many good examples to learn from.
+
+## Practical tips:
+
+You can observe the output of commands and functions that in some way use the
+macro system by setting this option to true : `set_option trace.Elab.definition
+true`
+
+Lean also offers the option of limiting the region in which option is set with
+the syntax `set_option ... in`):
+
+Hygiene can be disabled with the command option `set_option hygiene false`

doc/monads/.gitignore

@@ -0,0 +1 @@
+*.lean.md

doc/monads/applicatives.lean

@@ -0,0 +1,334 @@
+/-!
+# Applicative Functors
+
+Building on [Functors](functors.lean.md) is the [Applicative
+Functor](https://en.wikipedia.org/wiki/Applicative_functor). For simplicity, you can refer to these
+simply as "Applicatives". These are a little tricker than functors, but still simpler than monads.
+Let's see how they work!
+
+## What is an Applicative Functor?
+
+An applicative functor defines a default or "base" construction for an object and allows
+function application to be chained across multiple instances of the structure. All applicative
+functors are functors, meaning they must also support the "map" operation.
+
+## How are Applicatives represented in Lean?
+
+An [applicative functor](https://en.wikipedia.org/wiki/Applicative_functor) is an intermediate
+structure between `Functor` and `Monad`. It mainly consists of two operations:
+
+* `pure : α → F α`
+* `seq : F (α → β) → F α → F β` (written as `<*>`)
+
+The `pure` operator specifies how you can wrap a normal object `α` into an instance of this structure `F α`.
+This is the "default" mechanism mentioned above.
+
+The `seq` operator allows you to chain operations by wrapping a function in a structure. The name
+"applicative" comes from the fact that you "apply" functions from within the structure, rather than
+simply from outside the structure, as was the case with `Functor.map`.
+
+Applicative in Lean is built on some helper type classes, `Functor`, `Pure` and `Seq`:
+
+-/
+namespace hidden -- hidden
+class Applicative (f : Type u → Type v) extends Functor f, Pure f, Seq f, SeqLeft f, SeqRight f where
+  map      := fun x y => Seq.seq (pure x) fun _ => y
+  seqLeft  := fun a b => Seq.seq (Functor.map (Function.const _) a) b
+  seqRight := fun a b => Seq.seq (Functor.map (Function.const _ id) a) b
+end hidden -- hidden
+/-!
+Notice that as with `Functor` it is also a type transformer `(f : Type u → Type v)` and notice the
+`extends Functor f` is ensuring the base `Functor` also performs that same type transformation.
+
+As stated above, all applicatives are then functors. This means you can assume that `map` already
+exists for all these types.
+
+The `Pure` base type class is a very simple type class that supplies the `pure` function.
+
+-/
+namespace hidden -- hidden
+class Pure (f : Type u → Type v) where
+   pure {α : Type u} : α → f α
+end hidden -- hidden
+/-!
+
+You can think of it as lifting the result of a pure value to some monadic type. The simplest example
+of `pure` is the `Option` type:
+
+-/
+#eval (pure 10 : Option Nat)  -- some 10
+/-!
+
+Here we used the `Option` implementation of `pure` to wrap the `Nat 10` value in an `Option Nat`
+type resulting in the value `some 10`, and in fact if you look at the Monad instance of `Option` , you
+will see that `pure` is indeed implemented using `Option.some`:
+
+-/
+instance : Monad Option where
+    pure := Option.some
+/-!
+
+The `Seq` type class is also a simple type class that provides the `seq` operator which can
+also be written using the special syntax `<*>`.
+
+-/
+namespace hidden -- hidden
+class Seq (f : Type u → Type v) : Type (max (u+1) v) where
+  seq : {α β : Type u} → f (α → β) → (Unit → f α) → f β
+end hidden -- hidden
+/-!
+
+
+## Basic Applicative Examples
+
+Many of the basic functors also have instances of `Applicative`.
+For example, `Option` is also `Applicative`.
+
+So let's take a look and what the `seq` operator can do.  Suppose you want to multiply two `Option Nat`
+objects.  Your first attempt might be this:
+
+-/
+#check_failure (some 4) * (some 5)      -- failed to synthesize instance
+/-!
+
+You then might wonder how to use the `Functor.map` to solve this since you could do these before:
+
+-/
+#eval (some 4).map (fun x => x * 5)  -- some 20
+
+#eval (some 4).map (· * 5)  -- some 20
+
+#eval (· * 5) <$> (some 4)   -- some 20
+/-!
+
+Remember that `<$>` is the infix notation for `Functor.map`.
+
+The functor `map` operation can apply a multiplication to the value in the `Option` and then lift the
+result back up to become a new `Option` , but this isn't what you need here.
+
+The `Seq.seq` operator `<*>` can help since it can apply a function to the items inside a
+container and then lift the result back up to the desired type, namely `Option` .
+
+There are two ways to do this:
+
+-/
+#eval pure (.*.) <*> some 4 <*> some 5 -- some 20
+
+#eval (.*.) <$> some 4 <*> some 5 -- some 20
+/-!
+
+In the first way, we start off by wrapping the function in an applicative using pure. Then we apply
+this to the first `Option` , and again to the second `Option`  in a chain of operations.  So  you can see
+how `Seq.seq` can be chained in fact, `Seq.seq` is really all about chaining of operations.
+
+But in this case there is a simpler way.  In the second way, you can see that "applying" a single
+function to a container is the same as using `Functor.map`. So you use `<$>` to "transform" the first
+option into an `Option` containing a function, and then apply this function over the second value.
+
+Now if either side is `none`, the result is `none`, as expected, and in this case the
+`seq` operator was able to eliminate the multiplication:
+
+-/
+#eval (.*.) <$> none <*> some 5  -- none
+#eval (.*.) <$> some 4 <*> none  -- none
+/-!
+
+For a more interesting example, let's make `List` an applicative by adding the following
+definition:
+
+-/
+instance : Applicative List where
+  pure := List.singleton
+  seq f x := List.flatMap f fun y => Functor.map y (x ())
+/-!
+
+Notice you can now sequence a _list_ of functions and a _list_ of items.
+The trivial case of sequencing a singleton list is in fact the same as `map`, as you saw
+earlier with the `Option`  examples:
+
+-/
+#eval [ (·+2)] <*> [4, 6] -- [6, 8]
+#eval (·+2) <$> [4,6]    -- [6, 8]
+/-!
+
+But now with list it is easier to show the difference when you do this:
+
+-/
+#eval [(·+2), (· *3)] <*> [4, 6] -- [6, 8, 12, 18]
+/-!
+
+Why did this produce 4 values?  The reason is because `<*>` applies _every_ function to _every_
+value in a pairwise manner.  This makes sequence really convenient for solving certain problems. For
+example, how do you get the pairwise combinations of all values from two lists?
+
+-/
+#eval Prod.mk <$> [1, 2, 3] <*> [4, 5, 6]
+-- [(1, 4), (1, 5), (1, 6), (2, 4), (2, 5), (2, 6), (3, 4), (3, 5), (3, 6)]
+/-!
+
+How do you get the sum of these pairwise values?
+-/
+#eval (·+·) <$> [1, 2, 3] <*> [4, 5, 6]
+-- [5, 6, 7, 6, 7, 8, 7, 8, 9]
+/-!
+
+Here you can use `<$>` to "transform" each element of the first list into a function, and then apply
+these functions over the second list.
+
+If you have 3 lists, and want to find all combinations of 3 values across those lists you
+would need helper function that can create a tuple out of 3 values, and Lean provides a
+very convenient syntax for that `(·,·,·)`:
+
+-/
+#eval (·,·,·) <$> [1, 2] <*> [3, 4] <*> [5, 6]
+-- [(1, 3, 5), (1, 3, 6), (1, 4, 5), (1, 4, 6), (2, 3, 5), (2, 3, 6), (2, 4, 5), (2, 4, 6)]
+/-!
+
+And you could sum these combinations if you first define a sum function that takes three inputs and
+then you could chain apply this over the three lists.  Again lean can create such a function
+with the expression `(·+·+·)`:
+
+-/
+#eval (·+·+·) <$> [1, 2] <*> [3, 4] <*> [5, 6]
+-- [9, 10, 10, 11, 10, 11, 11, 12]
+/-!
+
+And indeed each sum here matches the expected values if you manually sum the triples we
+show above.
+
+**Side note:** there is another way to combine lists with a function that does not do the pairwise
+combinatorics, it is called `List.zipWith`:
+
+-/
+#eval List.zipWith (·+·) [1, 2, 3] [4, 5, 6]
+-- [5, 7, 9]
+/-!
+
+And there is a helper function named `List.zip` that calls `zipWith` using the function `Prod.mk`
+so you get a nice zipped list like this:
+
+-/
+#eval List.zip [1, 2, 3] [4, 5, 6]
+-- [(1, 4), (2, 5), (3, 6)]
+/-!
+
+And of course, as you would expect, there is an `unzip` also:
+
+-/
+#eval List.unzip (List.zip [1, 2, 3] [4, 5, 6])
+-- ([1, 2, 3], [4, 5, 6])
+/-!
+
+## Example: A Functor that is not Applicative
+
+From the chapter on [functors](functors.lean.md) you might remember this example of `LivingSpace`
+that had a `Functor` instance:
+
+-/
+structure LivingSpace (α : Type) where
+  totalSize : α
+  numBedrooms : Nat
+  masterBedroomSize : α
+  livingRoomSize : α
+  kitchenSize : α
+  deriving Repr, BEq
+
+def LivingSpace.map (f : α → β) (s : LivingSpace α) : LivingSpace β :=
+  { totalSize := f s.totalSize
+    numBedrooms := s.numBedrooms
+    masterBedroomSize := f s.masterBedroomSize
+    livingRoomSize := f s.livingRoomSize
+    kitchenSize := f s.kitchenSize }
+
+instance : Functor LivingSpace where
+  map := LivingSpace.map
+/-!
+
+It wouldn't really make sense to make an `Applicative` instance here. How would you write `pure` in
+the `Applicative` instance? By taking a single value and plugging it in for total size _and_ the
+master bedroom size _and_ the living room size? That wouldn't really make sense. And what would the
+numBedrooms value be for the default? What would it mean to "chain" two of these objects together?
+
+If you can't answer these questions very well, then it suggests this type isn't really an
+Applicative functor.
+
+## SeqLeft and SeqRight
+
+You may remember seeing the `SeqLeft` and `SeqRight` base types on `class Applicative` earlier.
+These provide the `seqLeft` and `seqRight` operations which also have some handy notation
+shorthands `<*` and `*>` respectively. Where: `x <* y` evaluates `x`, then `y`, and returns the
+result of `x` and `x *> y` evaluates `x`, then `y`,  and returns the result of `y`.
+
+To make it easier to remember, notice that it returns that value that the `<*` or `*>` notation is
+pointing at.  For example:
+
+-/
+#eval (some 1) *> (some 2) -- Some 2
+#eval (some 1) <* (some 2) -- Some 1
+/-!
+
+So these are a kind of "discard" operation.  Run all the actions, but only return the values that you
+care about. It will be easier to see these in action when you get to full Monads, but they are used
+heavily in the Lean `Parsec` parser combinator library where you will find parsing functions like
+this one which parses the XML declaration `<?xml version="1.0" encoding='utf-8' standalone="yes">`:
+
+```lean
+def XMLdecl : Parsec Unit := do
+  skipString "<?xml"
+  VersionInfo
+  optional EncodingDecl *> optional SDDecl *> optional S *> skipString "?>"
+```
+
+But you will need to understand full Monads before this will make sense.
+
+## Lazy Evaluation
+
+Diving a bit deeper, (you can skip this and jump to the [Applicative
+Laws](laws.lean.md#what-are-the-applicative-laws) if don't want to dive into this implementation detail right
+now). But, if you write a simple `Option` example `(.*.) <$> some 4 <*> some 5` that produces `some 20`
+using `Seq.seq` you will see something interesting:
+
+-/
+#eval Seq.seq ((.*.) <$> some 4) (fun (_ : Unit) => some 5) -- some 20
+/-!
+
+This may look a bit cumbersome, specifically, why did we need to invent this funny looking function
+`fun (_ : Unit) => (some 5)`?
+
+Well if you take a close look at the type class definition:
+```lean
+class Seq (f : Type u → Type v) where
+  seq : {α β : Type u} → f (α → β) → (Unit → f α) → f β
+```
+
+You will see this function defined here: `(Unit → f α)`, this is a function that takes `Unit` as input
+and produces the output of type `f α` where `f` is the container type `Type u -> Type v`, in this example `Option`
+and `α` is the element type `Nat`, so `fun (_ : Unit) => some 5` matches this definition because
+it is taking an input of type Unit and producing `some 5` which is type `Option Nat`.
+
+The that `seq` is defined this way is because Lean is an eagerly evaluated language
+(call-by-value), you have to use this kind of Unit function whenever you want to explicitly delay
+evaluation and `seq` wants that so it can eliminate unnecessary function evaluations whenever
+possible.
+
+Fortunately the `<*>` infix notation hides this from you by creating this wrapper function for you.
+If you look up the notation using F12 in VS Code you will find it contains `(fun _ : Unit => b)`.
+
+Now to complete this picture you will find the default implementation of `seq` on the Lean `Monad`
+type class:
+
+```lean
+class Monad (m : Type u → Type v) extends Applicative m, Bind m where
+  seq      f x := bind f fun y => Functor.map y (x ())
+```
+
+Notice here that `x` is the `(Unit → f α)` function, and it is calling that function by passing the
+Unit value `()`, which is the Unit value (Unit.unit).  All this just to ensure delayed evaluation.
+
+## How do Applicatives help with Monads?
+
+Applicatives are helpful for the same reasons as functors. They're a relatively simple abstract
+structure that has practical applications in your code. Now that you understand how chaining
+operations can fit into a structure definition, you're in a good position to start learning about
+[Monads](monads.lean.md)!
+-/

doc/monads/except.lean

@@ -0,0 +1,178 @@
+/-!
+# Except
+
+The `Except` Monad adds exception handling behavior to your functions.  Exception handling
+in other languages like Python or Java is done with a built in `throw` method that you
+can use anywhere.  In `Lean` you can only `throw` an exception when your function is
+executing in the context of an `Except` monad.
+
+-/
+def divide (x y: Float): Except String Float :=
+  if y == 0 then
+    throw "can't divide by zero"
+  else
+    pure (x / y)
+
+#eval divide 5 2  -- Except.ok 2.500000
+#eval divide 5 0  -- Except.error "can't divide by zero"
+
+/-!
+
+Just as the `read` operation was available from the `ReaderM` monad and the `get` and `set`
+operations came with the `StateM` monad, here you can see a `throw` operation is provided by the
+`Except` monad.
+
+So in Lean, `throw` is not available everywhere like it is in most imperative programming languages.
+You have to declare your function can throw by changing the type signature to `Except String Float`.
+This creates a function that might return an error of type `String` or it might return a value of
+type `Float` in the non-error case.
+
+Once your function is monadic you also need to use the `pure` constructor of the `Except` monad to
+convert the pure non-monadic value `x / y` into the required `Except` object.  See
+[Applicatives](applicatives.lean.md) for details on `pure`.
+
+Now this return typing would get tedious if you had to include it everywhere that you call this
+function, however, Lean type inference can clean this up. For example, you can define a test
+function that calls the `divide` function and you don't need to say anything here about the fact that
+it might throw an error, because that is inferred:
+-/
+def test := divide 5 0
+
+#check test     -- Except String Float
+/-!
+
+Notice the Lean compiler infers the required `Except String Float` type information for you.
+And now you can run this test and get the expected exception:
+
+-/
+#eval test      -- Except.error "can't divide by zero"
+/-!
+
+
+## Chaining
+
+Now as before you can build a chain of monadic actions that can be composed together using `bind (>>=)`:
+-/
+def square (x : Float) : Except String Float :=
+  if x >= 100 then
+    throw "it's absolutely huge"
+  else
+    pure (x * x)
+
+#eval divide 6 2 >>= square  -- Except.ok 9.000000
+#eval divide 6 0 >>= square  -- Except.error "can't divide by zero"
+#eval divide 100 1 >>= square  -- Except.error "it's absolutely huge"
+
+def chainUsingDoNotation := do
+  let r ← divide 6 0
+  square r
+
+#eval chainUsingDoNotation  -- Except.error "can't divide by zero"
+
+/-!
+Notice in the second `divide 6 0` the exception from that division was nicely propagated along
+to the final result and the square function was ignored in that case.  You can see why the
+`square` function was ignored if you look at the implementation of `Except.bind`:
+-/
+def bind (ma : Except ε α) (f : α → Except ε β) : Except ε β :=
+  match ma with
+  | Except.error err => Except.error err
+  | Except.ok v      => f v
+/-!
+
+Specifically notice that it only calls the next function `f v` in the `Except.ok`, and
+in the error case it simply passes the same error along.
+
+Remember also that you can chain the actions with implicit binding by using the `do` notation
+as you see in the `chainUsingDoNotation` function above.
+
+## Try/Catch
+
+Now with all good exception handling you also want to be able to catch exceptions so your program
+can continue on or do some error recovery task, which you can do like this:
+-/
+def testCatch :=
+  try
+    let r ← divide 8 0  -- 'r' is type Float
+    pure (toString r)
+  catch e =>
+    pure s!"Caught exception: {e}"
+
+#check testCatch -- Except String String
+/-!
+
+Note that the type inferred by Lean for this function is `Except String String` so unlike the
+`test` function earlier, this time Lean type inference has figured out that since the pure
+value `(toString r)` is of type `String`, then this function must have type `Except String String`
+so you don't have to explicitly state this. You can always hover your mouse over `testCatch`
+or use `#check testCatch` to query Lean interactively to figure out what type inference
+has decided.  Lean type inference makes life easy for you, so it's good to use it
+when you can.
+
+You can now see the try/catch working in this eval:
+-/
+#eval testCatch -- Except.ok "Caught exception: can't divide by zero"
+/-!
+
+Notice the `Caught exception:` wrapped message is returned, and that it is returned as an
+`Except.ok` value, meaning `testCatch` eliminated the error result as expected.
+
+So you've interleaved a new concept into your functions (exception handling) and the compiler is still
+able to type check everything just as well as it does for pure functions and it's been able to infer
+some things along the way to make it even easier to manage.
+
+Now you might be wondering why `testCatch` doesn't infer the return type `String`? Lean does this as a
+convenience since you could have a rethrow in or after the catch block. If you really want to stop
+the `Except` type from bubbling up you can unwrap it like this:
+
+-/
+def testUnwrap : String := Id.run do
+  let r ← divide 8 0 -- r is type Except String Float
+  match r with
+  | .ok a => toString a -- 'a' is type Float
+  | .error e => s!"Caught exception: {e}"
+
+#check testUnwrap -- String
+#eval testUnwrap -- "Caught exception: can't divide by zero"
+
+/-!
+
+The `Id.run` function is a helper function that executes the `do` block and returns the result where
+`Id` is the _identity monad_.  So `Id.run do` is a pattern you can use to execute monads in a
+function that is not itself monadic.  This works for all monads except `IO` which, as stated earlier,
+you cannot invent out of thin air, you must use the `IO` monad given to your `main` function.
+
+## Monadic functions
+
+You can also write functions that are designed to operate in the context of a monad.
+These functions typically end in upper case M like `List.forM` used below:
+-/
+
+def validateList (x : List Nat) (max : Nat): Except String Unit := do
+  x.forM fun a => do
+    if a > max then throw "illegal value found in list"
+
+#eval validateList [1, 2, 5, 3, 8] 10 -- Except.ok ()
+#eval validateList [1, 2, 5, 3, 8] 5 -- Except.error "illegal value found in list"
+
+/-!
+Notice here that the `List.forM` function passes the monadic context through to the inner function
+so it can use the `throw` function from the `Except` monad.
+
+The `List.forM` function is defined like this where `[Monad m]` means "in the context of a monad `m`":
+
+-/
+def forM [Monad m] (as : List α) (f : α → m PUnit) : m PUnit :=
+  match as with
+  | []      => pure ⟨⟩
+  | a :: as => do f a; List.forM as f
+/-!
+
+## Summary
+
+Now that you know all these different monad constructs, you might be wondering how you can combine
+them. What if there was some part of your state that you wanted to be able to modify (using the
+State monad), but you also needed exception handling. How can you get multiple monadic capabilities
+in the same function. To learn the answer, head to [Monad Transformers](transformers.lean.md).
+
+-/

doc/monads/functors.lean

@@ -0,0 +1,227 @@
+/-!
+# Functor
+
+A `Functor` is any type that can act as a generic container that allows you to transform the
+underlying values inside the container using a function, so that the values are all updated, but the
+structure of the container is the same. This is called "mapping".
+
+A List is one of the most basic examples of a `Functor`.
+
+A list contains zero or more elements of the same, underlying type.  When you `map` a function over
+a list, you create a new list with the same number of elements, where each has been transformed by
+the function:
+-/
+#eval List.map (λ x => toString x) [1,2,3] -- ["1", "2", "3"]
+
+-- you can also write this using dot notation on the List object
+#eval [1,2,3].map (λ x => toString x)  -- ["1", "2", "3"]
+
+/-!
+Here we converted a list of natural numbers (Nat) to a list of strings where the lambda function
+here used `toString` to do the transformation of each element. Notice that when you apply `map` the
+"structure" of the object remains the same, in this case the result is always a `List` of the same
+size.
+
+Note that in Lean a lambda function can be written using `fun` keyword or the unicode
+symbol `λ` which you can type in VS code using `\la `.
+
+List has a specialized version of `map` defined as follows:
+-/
+def map (f : α → β) : List α → List β
+  | []    => []
+  | a::as => f a :: map f as
+
+/-!
+This is a very generic `map` function that can take any function that converts `(α → β)` and use it
+to convert `List α → List β`. Notice the function call `f a` above, this application of `f` is
+producing the converted items for the new list.
+
+Let's look at some more examples:
+
+-/
+-- List String → List Nat
+#eval ["elephant", "tiger", "giraffe"].map (fun s => s.length)
+-- [8, 5, 7]
+
+-- List Nat → List Float
+#eval [1,2,3,4,5].map (fun s => (s.toFloat) ^ 3.0)
+-- [1.000000, 8.000000, 27.000000, 64.000000, 125.000000]
+
+--- List String → List String
+#eval ["chris", "david", "mark"].map (fun s => s.capitalize)
+-- ["Chris", "David", "Mark"]
+/-!
+
+Another example of a functor is the `Option` type. Option contains a value or nothing and is handy
+for code that has to deal with optional values, like optional command line arguments.
+
+Remember you can construct an Option using the type constructors `some` or `none`:
+
+-/
+#check some 5 -- Option Nat
+#eval some 5  -- some 5
+#eval (some 5).map (fun x => x + 1) -- some 6
+#eval (some 5).map (fun x => toString x) -- some "5"
+/-!
+
+Lean also provides a convenient short hand syntax for `(fun x => x + 1)`, namely `(· + 1)`
+using the middle dot unicode character which you can type in VS code using `\. `.
+
+-/
+#eval (some 4).map (· * 5)  -- some 20
+/-!
+
+The `map` function preserves the `none` state of the Option, so again
+map preserves the structure of the object.
+
+-/
+def x : Option Nat := none
+#eval x.map (fun x => toString x) -- none
+#check x.map (fun x => toString x) -- Option String
+/-!
+
+Notice that even in the `none` case it has transformed `Option Nat` into `Option String` as
+you see in the `#check` command.
+
+## How to make a Functor Instance?
+
+The `List` type is made an official `Functor` by the following type class instance:
+
+-/
+instance : Functor List where
+  map := List.map
+/-!
+
+Notice all you need to do is provide the `map` function implementation.  For a quick
+example, let's supposed you create a new type describing the measurements of a home
+or apartment:
+
+-/
+structure LivingSpace (α : Type) where
+  totalSize : α
+  numBedrooms : Nat
+  masterBedroomSize : α
+  livingRoomSize : α
+  kitchenSize : α
+  deriving Repr, BEq
+/-!
+
+Now you can construct a `LivingSpace` in square feet using floating point values:
+-/
+abbrev SquareFeet := Float
+
+def mySpace : LivingSpace SquareFeet :=
+  { totalSize := 1800, numBedrooms := 4, masterBedroomSize := 500,
+    livingRoomSize := 900, kitchenSize := 400 }
+/-!
+
+Now, suppose you want anyone to be able to map a `LivingSpace` from one type of measurement unit to
+another.  Then you would provide a `Functor` instance as follows:
+
+-/
+def LivingSpace.map (f : α → β) (s : LivingSpace α) : LivingSpace β :=
+  { totalSize := f s.totalSize
+    numBedrooms := s.numBedrooms
+    masterBedroomSize := f s.masterBedroomSize
+    livingRoomSize := f s.livingRoomSize
+    kitchenSize := f s.kitchenSize }
+
+instance : Functor LivingSpace where
+  map := LivingSpace.map
+/-!
+
+Notice this functor instance takes `LivingSpace` and not the fully qualified type `LivingSpace SquareFeet`.
+Notice below that `LivingSpace` is a function from Type to Type.  For example, if you give it type `SquareFeet`
+it gives you back the fully qualified type `LivingSpace SquareFeet`.
+
+-/
+#check LivingSpace -- Type → Type
+/-!
+
+So the `instance : Functor` then is operating on the more abstract, or generic `LivingSpace` saying
+for the whole family of types `LivingSpace α` you can map to `LivingSpace β` using the generic
+`LivingSpace.map` map function by simply providing a function that does the more primitive mapping
+from `(f : α → β)`.  So `LivingSpace.map` is a sort of function applicator.
+This is called a "higher order function" because it takes a function as input
+`(α → β)` and returns another function as output `F α → F β`.
+
+Notice that `LivingSpace.map` applies a function `f` to convert the units of all the LivingSpace
+fields, except for `numBedrooms` which is a count (and therefore is not a measurement that needs
+converting).
+
+So now you can define a simple conversion function, let's say you want square meters instead:
+
+-/
+abbrev SquareMeters := Float
+def squareFeetToMeters (ft : SquareFeet ) : SquareMeters := (ft / 10.7639104)
+/-!
+
+and now bringing it all together you can use the simple function `squareFeetToMeters` to map
+`mySpace` to square meters:
+
+-/
+#eval mySpace.map squareFeetToMeters
+/-
+{ totalSize := 167.225472,
+  numBedrooms := 4,
+  masterBedroomSize := 46.451520,
+  livingRoomSize := 83.612736,
+  kitchenSize := 37.161216 }
+  -/
+/-!
+
+Lean also defines custom infix operator `<$>` for `Functor.map` which allows you to write this:
+-/
+#eval (fun s => s.length) <$> ["elephant", "tiger", "giraffe"] -- [8, 5, 7]
+#eval (fun x => x + 1) <$> (some 5) -- some 6
+/-!
+
+Note that the infix operator is left associative which means it binds more tightly to the
+function on the left than to the expression on the right, this means you can often drop the
+parentheses on the right like this:
+
+-/
+#eval (fun x => x + 1) <$> some 5 -- some 6
+/-!
+
+Note that Lean lets you define your own syntax, so `<$>` is nothing special.
+You can define your own infix operator like this:
+
+-/
+infixr:100 " doodle " => Functor.map
+
+#eval (· * 5) doodle [1, 2, 3]  -- [5, 10, 15]
+
+/-!
+Wow, this is pretty powerful.  By providing a functor instance on `LivingSpace` with an
+implementation of the `map` function it is now super easy for anyone to come along and
+transform the units of a `LivingSpace` using very simple functions like `squareFeetToMeters`. Notice
+that squareFeetToMeters knows nothing about `LivingSpace`.
+
+## How do Functors help with Monads ?
+
+Functors are an abstract mathematical structure that is represented in Lean with a type class. The
+Lean functor defines both `map` and a special case for working on constants more efficiently called
+`mapConst`:
+
+```lean
+class Functor (f : Type u → Type v) : Type (max (u+1) v) where
+  map : {α β : Type u} → (α → β) → f α → f β
+  mapConst : {α β : Type u} → α → f β → f α
+```
+
+Note that `mapConst` has a default implementation, namely:
+`mapConst : {α β : Type u} → α → f β → f α := Function.comp map (Function.const _)` in the `Functor`
+type class.  So you can use this default implementation and you only need to replace it if
+your functor has a more specialized variant than this (usually the custom version is more performant).
+
+In general then, a functor is a function on types `F : Type u → Type v` equipped with an operator
+called `map` such that if you have a function `f` of type `α → β` then `map f` will convert your
+container type from `F α → F β`. This corresponds to the category-theory notion of
+[functor](https://en.wikipedia.org/wiki/Functor) in the special case where the category is the
+category of types and functions between them.
+
+Understanding abstract mathematical structures is a little tricky for most people. So it helps to
+start with a simpler idea like functors before you try to understand monads.  Building on
+functors is the next abstraction called [Applicatives](applicatives.lean.md).
+-/

doc/monads/intro.md

@@ -0,0 +1,63 @@
+# Monads
+
+Monads are used heavily in Lean, as they are also in Haskell. Monads come from the wonderful world
+of [Category Theory](https://en.wikipedia.org/wiki/Monad_%28category_theory%29).
+
+Monads in Lean are so similar to Haskell that this introduction to monads is heavily based on the
+similar chapter of the [Monday Morning Haskell](https://mmhaskell.com/monads/). Many thanks to
+the authors of that material for allowing us to reuse it here.
+
+Monads build on the following fundamental type classes which you will need to understand
+first before fully understanding monads. Shown in light blue are some concrete functors
+and monads that will also be covered in this chapter:
+
+![image](../images/monads.svg)
+
+This chapter is organized to give you a bottom up introduction to monads, starting with functors and
+applicative functors, you'll get an intuition for how these abstract structures work in Lean. Then
+you'll dive into monads and learn how to use some of the most useful built-in ones.
+
+## [Functor](functors.lean.md)
+A functor is a type class that provides a map function and the map function is something many
+people are already familiar with so this should be easy to follow.  Here you will see some
+concrete examples in action with `List` and `Option`.
+
+## [Applicative Functors](applicatives.lean.md)
+Applicatives are a little more difficult to understand than functors, but their functionality can
+still be summed up in a couple simple functions.  Here you will learn how to create an
+`Applicative List` and a completely custom `Applicative` type.
+
+## [Monads Tutorial](monads.lean.md)
+Now that you have an intuition for how abstract structures work, you'll examine some of the problems
+that functors and applicative functors don't help you solve. Then you'll learn the specifics of how
+to actually use monads with some examples using the `Option` monad and the all important `IO` monad.
+
+## [Reader Monad](readers.lean.md)
+Now that you understand the details of what makes a monadic structure work, in this section, you'll
+learn about one of the most useful built in monads `ReaderM`, which gives your programs a
+global read-only context.
+
+## [State Monad](states.lean.md)
+This section introduces the `StateM` monad. This monad allows you to access a particular type that you can
+both read from and write to. It opens the door to fully stateful programming, allowing you to do many
+of the things a function programming language supposedly "can't" do.
+
+## [Except Monad](except.lean.md)
+
+Similar to the `Option` monad the `Except` monad allows you to change the signature of a function so
+that it can return an `ok` value or an `error` and it provides the classic exception handling
+operations `throw/try/catch` so that your programs can do monad-based exception handling.
+
+## [Monad Transformers](transformers.lean.md)
+
+Now that you are familiar with all the above monads it is time to answer the question - how you can
+make them work together? After all, there are definitely times when you need multiple kinds of
+monadic behavior. This section introduces the concept of monad transformers, which allow you to
+combine multiple monads into one.
+
+## [Monad Laws](laws.lean.md)
+This section examines what makes a monad a legal monad. You could just implement your monadic type
+classes any way you want and write "monad" instances, but starting back with functors and
+applicative functors, you'll learn that all these structures have "laws" that they are expected to
+obey with respect to their behavior. You can make instances that don't follow these laws. But you do
+so at your peril, as other programmers will be very confused when they try to use them.

doc/monads/laws.lean

@@ -0,0 +1,322 @@
+/-!
+# Monad Laws
+
+In the previous sections you learned how to use [Functors](functors.lean.md),
+[Applicatives](applicatives.lean.md), and [Monads](monads.lean.md), and you played with some useful
+instances including [Option](monads.lean.md), [IO](monads.lean.md), [Reader](readers.lean.md),
+[State](states.lean.md) and [Except](except.lean.md) and you learned about composition using [Monad
+Transformers](transformers.lean.md).
+
+So far, you've learned the concrete details you need in order to _use_ monads in your Lean programs.
+But there's still one more important concept you need if you want to _create_ new functors,
+applicatives and monads. Namely, the notion of _structural "laws"_ -- rules that these type
+classes should follow in order to meet other programmers' expectations about your code.
+
+## Life without Laws
+
+Remember Lean represents each of these abstract structures by a type class. Each of these type classes
+has one or two main functions. So, as long as you implement those functions and it type checks, you
+have a new functor, applicative, or monad, right?
+
+Well not quite. Yes, your program will compile and you'll be able to use the instances. But this
+doesn't mean your instances follow the mathematical constructs. If they don't, your instances won't
+fulfill other programmers' expectations. Each type class has its own "laws". For instance, suppose
+you have the following Point Functor:
+-/
+structure Point (α : Type) where
+  x : α
+  y : α
+  deriving Repr, BEq
+
+def Point.map (f : α → β) (s : Point α) : Point β :=
+  { x := f s.y,  -- an example of something weird
+    y := f s.x }
+
+instance : Functor Point where
+  map := Point.map
+
+#eval (·+2) <$> (Point.mk 1 2) -- { x := 4, y := 3 }
+
+/-!
+This Point does something weird, when you `map` it because it transposes the `x` and `y` coordinates
+which is not what other people would expect from a `map` function.  In fact, it breaks the rules
+as you will see below.
+
+## What are the Functor laws?
+
+Functors have two laws: the _identity_ law, and the _composition_ law. These laws express behaviors that
+your functor instances should follow. If they don't, other programmers will be very confused at the
+effect your instances have on their program.
+
+The identity law says that if you "map" the identity function (`id`) over your functor, the
+resulting functor should be the same. A succinct way of showing this on a `List` functor is:
+
+-/
+def list1 := [1,2,3]
+
+#eval id <$> list1 == list1 -- true
+/-!
+
+Now let's try the same test on the `Point` functor:
+-/
+
+def p1 : Point Nat := (Point.mk 1 2)
+
+#eval id <$> p1 == p1 -- false
+
+/-!
+Oh, and look while the `List` is behaving well, the `Point` functor fails this identity test.
+
+The _composition_ law says that if you "map" two functions in succession over a functor, this
+should be the same as "composing" the functions and simply mapping that one super-function over the
+functor.  In Lean you can compose two functions using `Function.comp f g` (or the syntax `f ∘ g`,
+which you can type in VS code using `\o `) and you will get the same results from both of these
+showing that the composition law holds for `List Nat`:
+
+-/
+def double (x : Nat) := x + x
+def square (x : Nat) := x * x
+
+#eval double <$> (square <$> list1) -- [2, 8, 18]
+
+#eval (double <$> (square <$> list1)) == ((double ∘ square) <$> list1) -- true
+
+-- ok, what about the Point class?
+#eval double <$> (square <$> p1) -- { x := 2, y := 8 }
+#eval (double ∘ square) <$> p1   -- { x := 8, y := 2 }
+
+#eval double <$> (square <$> p1) == (double ∘ square) <$> p1  -- false
+/-!
+Note that composition also fails on the bad `Point` because the x/y transpose.
+
+As you can see this bad `Point` implementation violates both of the functor laws. In this case it
+would not be a true functor. Its behavior would confuse any other programmers trying to use it. You
+should take care to make sure that your instances make sense. Once you get a feel for these type
+classes, the likelihood is that the instances you'll create will follow the laws.
+
+You can also write a bad functor that passes one law but not the other like this:
+-/
+def bad_option_map {α β : Type u} : (α → β) → Option α → Option β
+  | _, _ => none
+
+instance : Functor Option where
+    map := bad_option_map
+
+def t1 : Option Nat := some 10
+
+#eval id <$> t1 == t1 -- false
+#eval double <$> (square <$> t1) == (double ∘ square) <$> t1  -- true
+/-!
+
+This fails the id law but obeys the composition law. Hopefully this explains the value of these
+laws, and you don't need to see any more bad examples!
+
+## What are the Applicative Laws?
+
+While functors have two laws, applicatives have four laws:
+
+- Identity
+- Homomorphism
+- Interchange
+- Composition
+
+### Identity
+
+`pure id <*> v = v`
+
+Applying the identity function through an applicative structure should not change the underlying
+values or structure.  For example:
+-/
+instance : Applicative List where
+  pure := List.singleton
+  seq f x := List.flatMap f fun y => Functor.map y (x ())
+
+#eval pure id <*> [1, 2, 3]  -- [1, 2, 3]
+/-!
+
+The `pure id` statement here is wrapping the identity function in an applicative structure
+so that you can apply that over the container `[1, 2, 3]` using the Applicative `seq` operation
+which has the notation `<*>`.
+
+To prove this for all values `v` and any applicative `m` you can write this theorem:
+-/
+example [Applicative m] [LawfulApplicative m] (v : m α) :
+  pure id <*> v = v :=
+  by simp -- Goals accomplished 🎉
+/-!
+
+### Homomorphism
+
+`pure f <*> pure x = pure (f x)`
+
+Suppose you wrap a function and an object in `pure`. You can then apply the wrapped function over the
+wrapped object. Of course, you could also apply the normal function over the normal object, and then
+wrap it in `pure`. The homomorphism law states these results should be the same.
+
+For example:
+
+-/
+def x := 1
+def f := (· + 2)
+
+#eval pure f <*> pure x = (pure (f x) : List Nat) -- true
+/-!
+
+You should see a distinct pattern here. The overriding theme of almost all these laws is that these
+`Applicative` types should behave like normal containers. The `Applicative` functions should not
+have any side effects. All they should do is facilitate the wrapping, unwrapping, and transformation
+of data contained in the container resulting in a new container that has the same structure.
+
+### Interchange
+
+`u <*> pure y = pure (. y) <*> u`.
+
+This law is a little more complicated, so don't sweat it too much. It states that the order that
+you wrap things shouldn't matter. One the left, you apply any applicative `u` over a pure wrapped
+object. On the right, you first wrap a function applying the object as an argument. Note that `(·
+y)` is short hand for: `fun f => f y`. Then you apply this to the first applicative `u`. These
+should be the same.
+
+For example:
+
+-/
+def y := 4
+def g : List (Nat → Nat) := [(· + 2)]
+
+#eval g <*> pure y = pure (· y) <*> g      -- true
+/-!
+
+You can prove this with the following theorem:
+-/
+example [Applicative m] [LawfulApplicative m] (u : m (α → β)) (y : α) :
+  u <*> pure y = pure (· y) <*> u :=
+  by simp [pure_seq] -- Goals accomplished 🎉
+
+/-!
+
+### Composition:
+
+`u <*> v <*> w = u <*> (v <*> w)`
+
+This final applicative law mimics the second functor law. It is a composition law. It states that
+function composition holds across applications within the applicative:
+
+For example:
+
+-/
+def u := [1, 2]
+def v := [3, 4]
+def w := [5, 6]
+
+#eval pure (·+·+·) <*> u <*> v <*> w
+-- [9, 10, 10, 11, 10, 11, 11, 12]
+
+#eval let grouping := pure (·+·) <*> v <*> w
+      pure (·+·) <*> u <*> grouping
+-- [9, 10, 10, 11, 10, 11, 11, 12]
+/-!
+
+To test composition you see the separate grouping `(v <*> w)` then that can be used in the outer
+sequence `u <*> grouping` to get the same final result `[9, 10, 10, 11, 10, 11, 11, 12]`.
+
+## What are the Monad Laws?
+
+Monads have three laws:
+
+- Left Identity
+- Right Identity
+- Associativity
+
+### Left Identity
+
+Identity laws for monads specify that `pure` by itself shouldn't really change anything about the
+structure or its values.
+
+Left identity is `x >>= pure = x` and is demonstrated by the following examples on a monadic `List`:
+-/
+instance : Monad List  where
+  pure := List.singleton
+  bind := List.flatMap
+
+def a := ["apple", "orange"]
+
+#eval a >>= pure      -- ["apple", "orange"]
+
+#eval a >>= pure = a  -- true
+
+/-!
+
+### Right Identity
+
+Right identity is `pure x >>= f = f x` and is demonstrated by the following example:
+-/
+def h (x : Nat) : Option Nat :=  some (x + 1)
+def z := 5
+
+#eval pure z >>= h                  -- some 6
+#eval h z                           -- some 6
+
+#eval pure z >>= h = h z            -- true
+/-!
+
+So in this example, with this specific `z` and `h`, you see that the rule holds true.
+
+
+### Associativity
+
+The associativity law is written as:
+```lean,ignore
+  x >>= f >>= g = x >>= (λ x => f x >>= g)
+```
+where `(x : m α)` and `(f : α → m β)` and `(g : β → m γ)`.
+
+The associativity law is difficult to parse like some of the applicative laws, but what it is saying
+is that if you change the grouping of `bind` operations, you should still get the same result.
+This law has a parallel structure to the other composition laws.
+
+You can see this in action in the following rewrite of `runOptionFuncsBind` from [monads](monads.lean.md):
+-/
+def optionFunc1 : String -> Option Nat
+  | "" => none
+  | str => some str.length
+
+def optionFunc2 (i : Nat) : Option Float :=
+  if i % 2 == 0 then none else some (i.toFloat * 3.14159)
+
+def optionFunc3 (f : Float) : Option (List Nat) :=
+  if f > 15.0 then none else some [f.floor.toUInt32.toNat, f.ceil.toUInt32.toNat]
+
+
+def runOptionFuncsBind (input : String) : Option (List Nat) :=
+   optionFunc1 input >>= optionFunc2 >>= optionFunc3
+
+def runOptionFuncsBindGrouped (input : String) : Option (List Nat) :=
+   optionFunc1 input >>= (λ x => optionFunc2 x >>= optionFunc3)
+
+#eval runOptionFuncsBind "big"        -- some [9, 10]
+#eval runOptionFuncsBindGrouped "big" -- some [9, 10]
+/-!
+
+Notice here we had to insert a `λ` function just like the definition says: `(λ x => f x >>= g)`.
+This is because unlike applicatives, you can't resolve the structure of later operations without the
+results of earlier operations quite as well because of the extra context monads provide. But you can
+still group their later operations into composite functions taking their inputs from earlier on, and
+the result should be the same.
+
+
+## Summary
+
+While these laws may be a bit difficult to understand just by looking at them, the good news is that
+most of the instances you'll make will naturally follow the laws so long as you keep it simple, so
+you shouldn't have to worry about them too much.
+
+There are two main ideas from all the laws:
+
+1. Applying the identity or pure function should not change the underlying values or structure.
+1. It should not matter what order you group operations in.  Another way to state this is function
+   composition should hold across your structures.
+
+Following these laws will ensure other programmers are not confused by the behavior of your
+new functors, applicatives and monads.
+
+-/

doc/monads/monads.lean

@@ -0,0 +1,300 @@
+/-!
+# Monads
+
+Building on [Functors](functors.lean.md) and [Applicatives](applicatives.lean.md) we can now
+introduce [monads](https://en.wikipedia.org/wiki/Monad_%28category_theory%29).
+
+A monad is another type of abstract, functional structure. Let's explore what makes it different
+from the first two structures.
+
+## What is a Monad?
+
+A monad is a computational context. It provides a structure that allows you to chain together
+operations that have some kind of shared state or similar effect. Whereas pure functional code can
+only operate on explicit input parameters and affect the program through explicit return values,
+operations in a monad can affect other computations in the chain implicitly through side effects,
+especially modification of an implicitly shared value.
+
+## How are monads represented in Lean?
+
+Like functors and applicatives, monads are represented with a type class in Lean:
+
+```lean,ignore
+class Monad (m : Type u → Type v) extends Applicative m, Bind m where
+```
+
+Just as every applicative is a functor, every monad is also an applicative and there's one more new
+base type class used here that you need to understand, namely, `Bind`.
+
+```lean,ignore
+class Bind (f : Type u → Type v) where
+  bind : {α β : Type u} → f α → (α → f β) → f β
+```
+
+The `bind` operator also has infix notation `>>=` where `x >>= g` represents the result of executing
+`x` to get a value of type `f α` then unwrapping the value `α` from that and passing it to function
+`g` of type `α → f β` returning the result of type `f β` where `f` is the target structure type
+(like `Option` or List)
+
+This `bind` operation looks similar to the other ones you've seen so far, if you put them all
+together `Monad` has the following operations:
+
+```lean,ignore
+class Monad (f : Type u → Type v) extends Applicative f, Bind f where
+  pure {α : Type u} : α → f α
+  map : {α β : Type u} → (α → β) → f α → f β
+  seq : {α β : Type u} → f (α → β) → (Unit → f α) → f β
+  bind : {α β : Type u} → f α → (α → f β) → f β
+  ...
+```
+
+Notice `Monad` also contains `pure` it must also have a "default" way to wrap a value in the
+structure.
+
+The `bind` operator is similar to the applicative `seq` operator in that it chains two operations,
+with one of them being function related. Notice that `bind`, `seq` and `map` all take a function of
+some kind.  Let's examine those function types:
+
+- map: `(α → β)`
+- seq: `f (α → β)`
+- bind: `(α → f β)`
+
+So `map` is a pure function, `seq` is a pure function wrapped in the structure, and `bind` takes a
+pure input but produces an output wrapped in the structure.
+
+Note: we are ignoring the `(Unit → f α)` function used by `seq` here since that has a special
+purpose explained in [Applicatives Lazy Evaluation](applicatives.lean.md#lazy-evaluation).
+
+## Basic Monad Example
+
+Just as `Option` is a functor and an applicative functor, it is also a monad! Let's start with how
+`Option` implements the Monad type class.
+
+-/
+instance : Monad Option where
+  pure := Option.some
+  bind := Option.bind
+/-!
+
+where:
+
+```lean,ignore
+def Option.bind : Option α → (α → Option β) → Option β
+  | none,   _ => none
+  | some a, f => f a
+```
+
+> **Side note**: this function definition is using a special shorthand syntax in Lean where the `:=
+match a, b with` code can be collapsed away. To make this more clear consider the following simpler
+example, where `Option.bind` is using the second form like `bar`:
+
+-/
+def foo (x : Option Nat) (y : Nat) : Option Nat :=
+  match x, y with
+  | none, _ => none
+  | some x, y => some (x + y)
+
+def bar : Option Nat → Nat → Option Nat
+  | none, _ => none
+  | some x, y => some (x + y)
+
+#eval foo (some 1) 2  -- some 3
+#eval bar (some 1) 2  -- some 3
+/-!
+What is important is that `Option.bind` is using a `match` statement to unwrap the input value
+`Option α`, if it is `none` then it does nothing and returns `none`, if it has a value of type `α`
+then it applies the function in the second argument `(α → Option β)` to this value, which is
+the expression `f a` that you see in the line `  | some a, f => f a` above.  The function
+returns a result of type `Option β` which then becomes the return value for `bind`.  So there
+is no structure wrapping required on the return value since the input function already did that.
+
+But let's bring in the definition of a monad. What does it mean to describe `Option` as a
+computational context?
+
+The `Option` monad encapsulates the context of failure. Essentially, the `Option` monad lets us
+abort a series of operations whenever one of them fails. This allows future operations to assume
+that all previous operations have succeeded. Here's some code to motivate this idea:
+
+-/
+def optionFunc1 : String -> Option Nat
+  | "" => none
+  | str => some str.length
+
+def optionFunc2 (i : Nat) : Option Float :=
+  if i % 2 == 0 then none else some (i.toFloat * 3.14159)
+
+def optionFunc3 (f : Float) : Option (List Nat) :=
+  if f > 15.0 then none else some [f.floor.toUInt32.toNat, f.ceil.toUInt32.toNat]
+
+def runOptionFuncs (input : String) : Option (List Nat) :=
+  match optionFunc1 input with
+  | none => none
+  | some i => match optionFunc2 i with
+    | none => none
+    | some f => optionFunc3 f
+
+#eval runOptionFuncs "big" -- some [9, 10]
+/-!
+
+Here you see three different functions that could fail. These are then combined in `runOptionFuncs`.
+But then you have to use nested `match` expressions to check if the previous result succeeded. It
+would be very tedious to continue this pattern much longer.
+
+The `Option` monad helps you fix this. Here's what this function looks like using the `bind`
+operator.
+
+-/
+
+def runOptionFuncsBind (input : String) : Option (List Nat) :=
+  optionFunc1 input >>= optionFunc2 >>= optionFunc3
+
+#eval runOptionFuncsBind "big" -- some [9, 10]
+/-!
+
+It's much cleaner now! You take the first result and pass it into the second and third functions
+using the `bind` operation. The monad instance handles all the failure cases so you don't have to!
+
+Let's see why the types work out. The result of `optionFunc1` input is simply `Option Nat`. Then the
+bind operator allows you to take this `Option Nat` value and combine it with `optionFunc2`, whose type
+is `Nat → Option Float` The **bind operator resolves** these to an `Option Float`. Then you pass this
+similarly through the bind operator to `optionFunc3`, resulting in the final type, `Option (List Nat)`.
+
+Your functions will not always combine so cleanly though. This is where `do` notation comes into play.
+This notation allows you to write monadic operations one after another, line-by-line. It almost makes
+your code look like imperative programming. You can rewrite the above as:
+-/
+
+def runOptionFuncsDo (input : String) : Option (List Nat) := do
+  let i ← optionFunc1 input
+  let f ← optionFunc2 i
+  optionFunc3 f
+
+#eval runOptionFuncsDo "big" -- some [9, 10]
+/-!
+
+The `←` operator used here is special. It effectively unwraps the value on the right-hand side from
+the monad. This means the value `i` has type `Nat`, _even though_ the result of `optionFunc1` is
+`Option Nat`. This is done using a `bind` operation under the hood.
+
+> Note you can use `<-` or the nice unicode symbol `←` which you can type into VS code by typing
+these characters `\l `.  When you type the final space, `\l` is replaced with `←`.
+
+Observe that we do not unwrap the final line of the computation. The function result is `Option
+(List Nat)` which matches what `optionFunc3` returns. At first glance, this may look more complicated
+than the `bind` example. However, it gives you a lot more flexibility, like mixing monadic and
+non-monadic statements, using if then/else structures with their own local do blocks and so on. It
+is particularly helpful when one monadic function depends on multiple previous functions.
+
+## Example using List
+
+You can easily make `List` into a monad with the following, since List already provides an
+implementation of `pure` and `bind`.
+
+-/
+instance : Monad List  where
+  pure := List.singleton
+  bind := List.flatMap
+/-!
+
+Like you saw with the applicative `seq` operator, the `bind` operator applies the given function
+to every element of the list.  It is useful to look at the bind implementation for List:
+
+-/
+open List
+def bind (a : List α) (b : α → List β) : List β := join (map b a)
+/-!
+
+So `Functor.map` is used to apply the function `b` to every element of `a` but this would
+return a whole bunch of little lists, so `join` is used to turn those back into a single list.
+
+Here's an example where you use `bind` to convert a list of strings into a combined list of chars:
+
+-/
+
+#eval "apple".toList  -- ['a', 'p', 'p', 'l', 'e']
+
+#eval ["apple", "orange"] >>= String.toList
+-- ['a', 'p', 'p', 'l', 'e', 'o', 'r', 'a', 'n', 'g', 'e']
+
+/-!
+
+
+## The IO Monad
+
+The `IO Monad` is perhaps the most important monad in Lean. It is also one of the hardest monads to
+understand starting out. Its actual implementation is too intricate to discuss when first learning
+monads. So it is best to learn by example.
+
+What is the **computational context** that describes the IO monad? IO operations can read
+information from or write information to the terminal, file system, operating system, and/or
+network. They interact with systems outside of your program. If you want to get user input, print a
+message to the user, read information from a file, or make a network call, you'll need to do so
+within the IO Monad.
+
+The state of the world outside your program can change at virtually any moment, and so this IO
+context is particularly special. So these IO operations are "side effects" which means you cannot
+perform them from "pure" Lean functions.
+
+Now, the most important job of pretty much any computer program is precisely to perform this
+interaction with the outside world. For this reason, the root of all executable Lean code is a
+function called main, with the type `IO Unit`. So every program starts in the IO monad!
+
+When your function is `IO` monadic, you can get any input you need, call into "pure" code with the
+inputs, and then output the result in some way. The reverse does not work. You cannot call into IO
+code from pure code like you can call into a function that takes `Option` as input. Another way to
+say this is you cannot invent an `IO` context out of thin air, it has to be given to you in your
+`main` function.
+
+Let's look at a simple program showing a few of the basic IO functions. It also uses `do` notation
+to make the code read nicely:
+-/
+def main : IO Unit := do
+  IO.println "enter a line of text:"
+  let stdin ← IO.getStdin            -- IO IO.FS.Stream (monadic)
+  let input ← stdin.getLine          -- IO.FS.Stream → IO String (monadic)
+  let uppercased := input.toUpper    -- String → String (pure)
+  IO.println uppercased              -- IO Unit (monadic)
+/-!
+
+So, once again you can see that the `do` notation lets you chain a series of monadic actions.
+`IO.getStdin` is of type `IO IO.FS.Stream` and `stdin.getLine` is of type `IO String`
+and `IO.println` is of type `IO Unit`.
+
+In between you see a non-monadic expression `let uppercased := input.toUpper` which is fine too.
+A let statement can occur in any monad. Just as you could unwrap `i` from `Option Nat` to get the
+inner Nat, you can use `←` to unwrap the result of `getLine` to get a String. You can then manipulate
+this value using normal pure string functions like `toUpper`, and then you can pass the result to the
+`IO.println` function.
+
+This is a simple echo program. It reads a line from the terminal, and then prints the line back out
+capitalized to the terminal. Hopefully it gives you a basic understanding of how IO works.
+
+You can test this program using `lean --run` as follows:
+
+```
+> lean --run Main.lean
+enter a line of text:
+the quick brown fox
+THE QUICK BROWN FOX
+```
+
+Here the user entered the string `the quick brown fox` and got back the uppercase result.
+
+## What separates Monads from Applicatives?
+
+The key that separates these is **context**. You cannot really determine the structure of
+"future" operations without knowing the results of "past" operations, because the past can alter the
+context in which the future operations work. With applicatives, you can't get the final function
+result without evaluating everything, but you can determine the structure of how the operation will
+take place. This allows some degree of parallelism with applicatives that is not generally possible
+with monads.
+
+
+## Conclusion
+
+Hopefully you now have a basic level understanding of what a monad is. But perhaps some more
+examples of what a "computational context" means would be useful to you. The Reader, State and
+Except monads each provide a concrete and easily understood context that can be compared easily to
+function parameters. You can learn more about those in [Reader monads](readers.lean.md),
+[State monads](states.lean.md), and the [Except monad](except.lean.md).
+-/

doc/monads/readers.lean

@@ -0,0 +1,199 @@
+/-!
+# Readers
+
+In the [previous section](monads.lean.md) you learned about the conceptual idea of monads. You learned
+what they are, and saw how some common types like `IO` and `Option` work as monads. Now in this
+section, you will be looking at some other useful monads. In particular, the `ReaderM` monad.
+
+## How to do Global Variables in Lean?
+
+In Lean, your code is generally "pure", meaning functions can only interact with the arguments
+passed to them. This effectively means you cannot have global variables. You can have global
+definitions, but these are fixed at compile time. If some user behavior might change them, you would have
+to wrap them in the `IO` monad, which means they can't be used from pure code.
+
+Consider this example. Here, you want to have an `Environment` containing different parameters as a
+global variable. However, you want to load these parameters from the process environment variables,
+which requires the `IO` monad.
+-/
+
+structure Environment where
+  path : String
+  home : String
+  user : String
+  deriving Repr
+
+def getEnvDefault (name : String): IO String := do
+  let val? ← IO.getEnv name
+  pure <| match val? with
+          | none => ""
+          | some s => s
+
+def loadEnv : IO Environment := do
+  let path ← getEnvDefault "PATH"
+  let home ← getEnvDefault "HOME"
+  let user ← getEnvDefault "USER"
+  pure { path, home, user }
+
+def func1 (e : Environment) : Float :=
+  let l1 := e.path.length
+  let l2 := e.home.length * 2
+  let l3 := e.user.length * 3
+  (l1 + l2 + l3).toFloat * 2.1
+
+def func2 (env : Environment) : Nat :=
+  2 + (func1 env).floor.toUInt32.toNat
+
+def func3 (env : Environment) : String :=
+  "Result: " ++ (toString (func2 env))
+
+def main : IO Unit := do
+  let env ← loadEnv
+  let str := func3 env
+  IO.println str
+
+#eval main -- Result: 7538
+
+/-!
+The only function actually using the environment is func1. However func1 is a pure function. This
+means it cannot directly call loadEnv, an impure function in the IO monad. This means the
+environment has to be passed through as a variable to the other functions, just so they can
+ultimately pass it to func1. In a language with global variables, you could save env as a global
+value in main. Then func1 could access it directly. There would be no need to have it as a parameter
+to func1, func2 and func3. In larger programs, these "pass-through" variables can cause a lot of
+headaches.
+
+## The Reader Solution
+
+The `ReaderM` monad solves this problem. It effectively creates a global read-only value of a
+specified type. All functions within the monad can "read" the type. Let's look at how the `ReaderM`
+monad changes the shape of this code. Now the functions **no longer need** to be given the
+`Environment` as an explicit parameter, as they can access it through the monad.
+-/
+
+def readerFunc1 : ReaderM Environment Float := do
+  let env ← read
+  let l1 := env.path.length
+  let l2 := env.home.length * 2
+  let l3 := env.user.length * 3
+  return (l1 + l2 + l3).toFloat * 2.1
+
+def readerFunc2 : ReaderM Environment Nat :=
+  readerFunc1 >>= (fun x => return 2 + (x.floor.toUInt32.toNat))
+
+def readerFunc3 : ReaderM Environment String := do
+  let x ← readerFunc2
+  return "Result: " ++ toString x
+
+def main2 : IO Unit := do
+  let env ← loadEnv
+  let str := readerFunc3.run env
+  IO.println str
+
+#eval main2 -- Result: 7538
+/-!
+The `ReaderM` monad provides a `run` method and it is the `ReaderM` run method that takes the initial
+`Environment` context.  So here you see `main2` loads the environment as before, and establishes
+the `ReaderM` context by passing `env` to the `run` method.
+
+> **Side note 1**: The `return` statement used above also needs some explanation.  The `return`
+statement in Lean is closely related to `pure`, but a little different. First the similarity is that
+`return` and `pure` both lift a pure value up to the Monad type. But `return` is a keyword so you do
+not need to parenthesize the expression like you do when using `pure`.  (Note: you can avoid
+parentheses when using `pure` by using the `<|` operator like we did above in the initial
+`getEnvDefault` function).  Furthermore, `return` can also cause an early `return` in a monadic
+function similar to how it can in an imperative language while `pure` cannot.
+
+> So technically if `return` is the last statement in a function it could be replaced with `pure <|`,
+but one could argue that `return` is still a little easier for most folks to read, just so long as
+you understand that `return` is doing more than other languages, it is also wrapping pure values in
+the monadic container type.
+
+> **Side note 2**: If the function `readerFunc3` also took some explicit arguments then you would have
+to write `(readerFunc3 args).run env` and this is a bit ugly, so Lean provides an infix operator
+`|>` that eliminates those parentheses so you can write `readerFunc3 args |>.run env` and then you can
+chain multiple monadic actions like this `m1 args1 |>.run args2 |>.run args3` and this is the
+recommended style.  You will see this pattern used heavily in Lean code.
+
+The `let env ← read` expression in `readerFunc1` unwraps the environment from the `ReaderM` so we
+can use it. Each type of monad might provide one or more extra functions like this, functions that
+become available only when you are in the context of that monad.
+
+Here the `readerFunc2` function uses the `bind` operator `>>=` just to show you that there are bind
+operations happening here.  The `readerFunc3` function uses the `do` notation you learned about in
+[Monads](monads.lean.md) which hides that bind operation and can make the code look cleaner.
+So the expression `let x ← readerFunc2` is also calling the `bind` function under the covers,
+so that you can access the unwrapped value `x` needed for the `toString x` conversion.
+
+The important difference here to the earlier code is that `readerFunc3` and `readerFunc2` no longer
+have an **explicit** Environment input parameter that needs to be passed along all the way to
+`readerFunc1`.  Instead, the `ReaderM` monad is taking care of that for you, which gives you the
+illusion of something like global context where the context is now available to all functions that use
+the `ReaderM` monad.
+
+The above code also introduces an important idea. Whenever you learn about a monad "X", there's
+often (but not always) a `run` function to execute that monad, and sometimes some additional
+functions like `read` that interact with the monad context.
+
+You might be wondering, how does the context actually move through the `ReaderM` monad? How can you
+add an input argument to a function by modifying its return type?  There is a special command in
+Lean that will show you the reduced types:
+-/
+#reduce (types := true) ReaderM Environment String   -- Environment → String
+/-!
+And you can see here that this type is actually a function!  It's a function that takes an
+`Environment` as input and returns a `String`.
+
+Now, remember in Lean that a function that takes an argument of type `Nat` and returns a `String`
+like `def f (a : Nat) : String` is the same as this function `def f : Nat → String`.  These are
+exactly equal as types.  Well this is being used by the `ReaderM` Monad to add an input argument to
+all the functions that use the `ReaderM` monad and this is why `main` is able to start things off by
+simply passing that new input argument in `readerFunc3.run env`. So now that you know the implementation
+details of the `ReaderM` monad you can see that what it is doing looks very much like the original
+code we wrote at the beginning of this section, only it's taking a lot of the tedious work off your
+plate and it is creating a nice clean separation between what your pure functions are doing, and the
+global context idea that the `ReaderM` adds.
+
+## withReader
+
+One `ReaderM` function can call another with a modified version of the `ReaderM` context. You can
+use the `withReader` function from the `MonadWithReader` type class to do this:
+
+-/
+def readerFunc3WithReader : ReaderM Environment String := do
+  let x ← withReader (λ env => { env with user := "new user" }) readerFunc2
+  return "Result: " ++ toString x
+
+/-!
+Here we changed the `user` in the `Environment` context to "new user" and then we passed that
+modified context to `readerFunc2`.
+
+So `withReader f m` executes monad `m` in the `ReaderM` context modified by `f`.
+
+## Handy shortcut with (← e)
+
+If you use the operator `←` in a let expression and the variable is only used once you can
+eliminate the let expression and place the `←` operator in parentheses like this
+call to loadEnv:
+-/
+def main3 : IO Unit := do
+  let str := readerFunc3 (← loadEnv)
+  IO.println str
+/-!
+
+## Conclusion
+
+It might not seem like much has been accomplished with this `ReaderM Environment` monad, but you will
+find that in larger code bases, with many different types of monads all composed together this
+greatly cleans up the code. Monads provide a beautiful functional way of managing cross-cutting
+concerns that would otherwise make your code very messy.
+
+Having this control over the inherited `ReaderM` context via `withReader` is actually very useful
+and something that is quite messy if you try and do this sort of thing with global variables, saving
+the old value, setting the new one, calling the function, then restoring the old value, making sure
+you do that in a try/finally block and so on. The `ReaderM` design pattern avoids that mess
+entirely.
+
+Now it's time to move on to [StateM Monad](states.lean.md) which is like a `ReaderM` that is
+also updatable.
+-/

doc/monads/states.lean

@@ -0,0 +1,265 @@
+import Lean.Data.HashMap
+/-!
+# State
+
+In the [previous section](readers.lean.md), you learned about the `ReaderM` monad. Hopefully this gave you
+a new perspective on Lean. It showed that, in fact, you _can_ have global variables of some sort;
+you just need to encode them in the type signature somehow, and this is what monads are for! In this
+part, you will explore the `StateM` monad, which is like a `ReaderM` only the state can also be updated.
+
+## Motivating example: Tic Tac Toe
+
+For this section, let's build a simple model for a Tic Tace Toe game. The main object is the `GameState`
+data type containing several important pieces of information. First and foremost, it has the
+"board", a map from 2D tile indices to the "Tile State" (X, O or empty). Then it also knows the
+current player, and it has a random generator.
+-/
+
+open Batteries (HashMap)
+abbrev TileIndex := Nat × Nat -- a 2D index
+
+inductive TileState where
+  | TileEmpty | TileX | TileO
+  deriving Repr, BEq
+
+inductive Player where
+  | XPlayer | OPlayer
+  deriving Repr, BEq
+
+abbrev Board := HashMap TileIndex TileState
+
+structure GameState where
+  board : Board
+  currentPlayer : Player
+  generator : StdGen
+
+/-!
+Let's think at a high level about how some of the game functions would work. You could, for
+instance, have a function for selecting a random move. This would output a `TileIndex` to play and
+alter the game's number generator. You would then make a move based on the selected move and the
+current player. This would change the board state as well as swap the current player. In other
+words, you have operations that depend on the current state of the game, but also need to **update
+that state**.
+
+## The StateM Monad to the Rescue
+
+This is exactly the situation the `StateM` monad deals with. The `StateM` monad wraps computations in
+the context of reading and modifying a global state object.
+
+It is parameterized by a single type parameter `s`, the state type in use. So just like the `ReaderM`
+has a single type you read from, the `StateM` has a single type you can both **read from and write
+to**. There are three primary actions you can take within the `StateM`monad:
+
+- **get** - retrieves the state, like Reader.read
+- **set** - updates the state
+- **modifyGet** - retrieves the state, then updates it
+
+There is also a `run` function, similar to `run` on `ReaderM`. Like the `ReaderM` monad, you must
+provide an initial state, in addition to the computation to run. `StateM` then produces two outputs:
+the result of the computation combined with the final updated state.
+
+If you wish to discard the final state and just get the computation's result, you can use
+`run'` method instead.  Yes in Lean, the apostrophe can be part of a name, you read this "run
+prime", and the general naming convention is that the prime method discards something.
+
+So for your Tic Tac Toe game, many of your functions will have a signature like `State GameState a`.
+
+## Stateful Functions
+
+Now you can examine some of the different functions mentioned above and determine their types.
+You can, for instance, pick a random move:
+
+-/
+open TileState
+
+def findOpen : StateM GameState (List TileIndex) := do
+  let game ← get
+  return game.board.toList.filterMap fun (i, x) => guard (x == TileEmpty) *> pure i
+
+def chooseRandomMove : StateM GameState TileIndex := do
+  let game ← get
+  let openSpots ← findOpen
+  let gen := game.generator
+  let (i, gen') := randNat gen 0 (openSpots.length - 1)
+  set { game with generator := gen' }
+  return openSpots[i]!
+
+/-!
+This returns a `TileIndex` and modifies the random number generator stored in the `GameState`!
+Notice you have a fun little use of the `Applicative.seqRight` operator `*>` in `findOpen`
+as described in [Applicatives](applicatives.lean.md).
+
+Now you can create the function that can make a move:
+-/
+open Player
+
+def tileStateForPlayer : Player → TileState
+| XPlayer => TileX
+| OPlayer => TileO
+
+def nextPlayer : Player →  Player
+| XPlayer => OPlayer
+| OPlayer => XPlayer
+
+def applyMove (i : TileIndex): StateM GameState Unit := do
+  let game ← get
+  let p := game.currentPlayer
+  let newBoard := game.board.insert i (tileStateForPlayer p)
+  set { game with currentPlayer := nextPlayer p, board := newBoard }
+
+/-!
+This updates the board in the `GameState` with the new tile, and then changes the current player,
+providing no output (`Unit` return type).
+
+So finally, you can combine these functions together with `do` notation, and it actually looks quite
+clean! You don't need to worry about the side effects. The different monadic functions handle them.
+Here's a sample of what your function might look like to play one turn of the game. At the end, it
+returns a boolean determining if all the spaces have been filled.
+
+Notice in `isGameDone` and `nextTurn` we have stopped providing the full return type
+`StateM GameState Unit`.  This is because Lean is able to infer the correct monadic return type
+from the context and as a result the code is now looking really clean.
+-/
+
+def isGameDone := do
+  return (← findOpen).isEmpty
+
+def nextTurn := do
+  let i ← chooseRandomMove
+  applyMove i
+  isGameDone
+
+/-!
+To give you a quick test harness that runs all moves for both players you can run this:
+-/
+
+def initBoard : Board := Id.run do
+  let mut board := HashMap.empty
+  for i in [0:3] do
+    for j in [0:3] do
+      let t : TileIndex := (i, j)
+      board := board.insert t TileEmpty
+  board
+
+def printBoard (board : Board) : IO Unit := do
+  let mut row : List String := []
+  for i in board.toList do
+    let s := match i.2 with
+      | TileEmpty => " "
+      | TileX => "X"
+      | TileO => "O"
+    row := row.append [s]
+    if row.length == 3 then
+      IO.println row
+      row := []
+
+def playGame := do
+  while true do
+    let finished ← nextTurn
+    if finished then return
+
+def main : IO Unit := do
+  let gen ← IO.stdGenRef.get
+  let (x, gen') := randNat gen 0 1
+  let gs := {
+    board := initBoard,
+    currentPlayer := if x = 0 then XPlayer else OPlayer,
+    generator := gen' }
+  let (_, g) := playGame |>.run gs
+  printBoard g.board
+
+#eval main
+-- [X, X, O]
+-- [X, O, O]
+-- [O, O, X]
+
+/-!
+
+Note that when you run the above code interactively the random number generator always starts in the
+same place.  But if you run `lean --run states.lean` then you will see randomness in the result.
+
+## Implementation
+
+It may be helpful to see how the `StateM` monad adds the input state and output state.  If you look
+at the reduced Type for `nextTurn`:
+-/
+#reduce StateM GameState Bool
+-- GameState → Bool × GameState
+/-!
+
+So a function like `nextTurn` that might have just returned a `Bool` has been modified by the
+`StateM` monad such that the initial `GameState` is passed in as a new input argument, and the output
+value has been changed to the pair `Bool × GameState` so that it can return the pure `Bool` and the
+updated `GameState`.  So `playGame` then is automatically saving that updated game state so that each
+time around the `while` loop it is acting on the new state, otherwise that would be an infinite loop!
+
+It is also interesting to see how much work the `do` and `←` notation are doing for you.  To
+implement the `nextTurn` function without these you would have to write this, manually plumbing
+the state all the way through:
+-/
+def nextTurnManually : StateM GameState Bool
+| state =>
+  let (i, gs) := chooseRandomMove |>.run state
+  let (_, gs') := applyMove i  |>.run gs
+  let (result, gs'') := isGameDone |>.run gs'
+  (result, gs'')
+
+/-!
+
+This expression `let (i, gs)` conveniently breaks a returned pair up into 2 variables.
+In the expression `let (_, gs')` we didn't care what the first value was so we used underscore.
+Notice that nextTurn is capturing the updated game state from `chooseRandomMove` in the variable
+`gs`, which it is then passing to `applyMove` which returns `gs'` which is passed to `isGameDone`
+and that function returns `gs''` which we then return from `nextTurnManually`.  Phew, what a lot
+of work you don't have to do when you use `do` notation!
+
+## StateM vs ReaderM
+
+While `ReaderM` functions can use `withReader` to modify the context before calling another function,
+`StateM` functions are a little more powerful, let's look at this function again:
+```
+def nextTurn : StateM GameState Bool := do
+  let i ← chooseRandomMove
+  applyMove i
+  isGameDone
+```
+
+In this function `chooseRandomMove` is modifying the state that `applyMove` is getting
+and `chooseRandomMove` knows nothing about `applyMove`.  So `StateM` functions can have this
+kind of downstream effect outside their own scope, whereas, `withReader` cannot do that.
+
+So there is no equivalent to `withReader` for `StateM`, besides you can always use the `StateM`
+`set` function to modify the state before calling the next function anyway.  You could however,
+manually call a `StateM` function like you see in `nextTurnManually` and completely override
+the state at any point that way.
+
+## State, IO and other languages
+
+When thinking about Lean, it is often seen as a restriction that you can't have global variables or
+`static` variables like you can with other languages like Python or C++. However, hopefully you see
+now this isn't true. You can have a data type with exactly the same functionality as a Python class.
+You would simply have many functions that can modify some global state using the `StateM` monad.
+
+The difference is in Lean you simply put a label on these types of functions. You don't allow it to
+happen for free anywhere in an uncontrolled fashion because that results in too many sleepless
+nights debugging nasty code. You want to know when side effects can potentially happen, because
+knowing when they can happen makes your code easier to reason about. In a Python class, many of the
+methods won't actually need to modify the global state. But they could, which makes it harder to
+debug them. In Lean you can simply make these pure functions, and the compiler will ensure they stay
+pure and cannot modify any global state.
+
+IO is the same way. It's not like you can't perform IO in Lean. Instead, you want to label the areas
+where you can, to increase your certainty about the areas where you don't need to. When you know part of
+your code cannot communicate with the outside world, you can be far more certain of its behavior.
+
+The `StateM` monad is also a more disciplined way of managing side effects. Top level code could
+call a `StateM` function multiple times with different independent initial states, even doing that
+across multiple tasks in parallel and each of these cannot clobber the state belonging to other
+tasks. Monadic code is more predictable and reusable than code that uses global variables.
+
+## Summary
+
+That wraps it up for the `StateM` monad! There is one more very useful monad that can be used to do
+exception handling which will be covered in the [next section](except.lean.md).
+
+-/

doc/monads/transformers.lean

@@ -0,0 +1,316 @@
+/-!
+# Monad Transformers
+
+In the previous sections you learned about some handy monads [Option](monads.lean.md),
+[IO](monads.lean.md), [Reader](readers.lean.md), [State](states.lean.md) and
+[Except](except.lean.md), and you now know how to make your function use one of these, but what you
+do not yet know is how to make your function use multiple monads at once.
+
+For example, suppose you need a function that wants to access some Reader context and optionally throw
+an exception?  This would require composition of two monads `ReaderM` and `Except` and this is what
+monad transformers are for.
+
+A monad transformer is fundamentally a wrapper type. It is generally parameterized by another
+monadic type. You can then run actions from the inner monad, while adding your own customized
+behavior for combining actions in this new monad. The common transformers add `T` to the end of an
+existing monad name. You will find `OptionT`, `ExceptT`, `ReaderT`, `StateT` but there is no transformer
+for `IO`.  So generally if you need `IO` it becomes the innermost wrapped monad.
+
+In the following example we use `ReaderT` to provide some read only context to a function
+and this `ReaderT` transformer will wrap an `Except` monad.  If all goes well the
+`requiredArgument` returns the value of a required argument and `optionalSwitch`
+returns true if the optional argument is present.
+
+-/
+abbrev Arguments := List String
+
+def indexOf? [BEq α] (xs : List α) (s : α) (start := 0): Option Nat :=
+  match xs with
+  | [] => none
+  | a :: tail => if a == s then some start else indexOf? tail s (start+1)
+
+def requiredArgument (name : String) : ReaderT Arguments (Except String) String := do
+  let args ← read
+  let value := match indexOf? args name with
+    | some i => if i + 1 < args.length then args[i+1]! else ""
+    | none => ""
+  if value == "" then throw s!"Command line argument {name} missing"
+  return value
+
+def optionalSwitch (name : String) : ReaderT Arguments (Except String) Bool := do
+  let args ← read
+  return match (indexOf? args name) with
+  | some _ => true
+  | none => false
+
+#eval requiredArgument "--input" |>.run ["--input", "foo"]
+-- Except.ok "foo"
+
+#eval requiredArgument "--input" |>.run ["foo", "bar"]
+-- Except.error "Command line argument --input missing"
+
+#eval optionalSwitch "--help" |>.run ["--help"]
+-- Except.ok true
+
+#eval optionalSwitch "--help" |>.run []
+-- Except.ok false
+
+/-!
+Notice that `throw` was available from the inner `Except` monad. The cool thing is you can switch
+this around and get the exact same result using `ExceptT` as the outer monad transformer and
+`ReaderM` as the wrapped monad. Try changing requiredArgument to `ExceptT String (ReaderM Arguments) Bool`.
+
+Note: the `|>.` notation is described in [Readers](readers.lean.md#the-reader-solution).
+
+## Adding more layers
+
+Here's the best part about monad transformers. Since the result of a monad transformer is itself a
+monad, you can wrap it inside another transformer! Suppose you need to pass in some read only context
+like the command line arguments, update some read-write state (like program Config) and optionally
+throw an exception, then you could write this:
+
+-/
+structure Config where
+  help : Bool := false
+  verbose : Bool := false
+  input : String := ""
+  deriving Repr
+
+abbrev CliConfigM := StateT Config (ReaderT Arguments (Except String))
+
+def parseArguments : CliConfigM Bool := do
+  let mut config ← get
+  if (← optionalSwitch "--help") then
+    throw "Usage: example [--help] [--verbose] [--input <input file>]"
+  config := { config with
+    verbose := (← optionalSwitch "--verbose"),
+    input := (← requiredArgument "--input") }
+  set config
+  return true
+
+def main (args : List String) : IO Unit := do
+  let config : Config := { input := "default"}
+  match parseArguments |>.run config |>.run args with
+  | Except.ok (_, c) => do
+    IO.println s!"Processing input '{c.input}' with verbose={c.verbose}"
+  | Except.error s => IO.println s
+
+
+#eval main ["--help"]
+-- Usage: example [--help] [--verbose] [--input <input file>]
+
+#eval main ["--input", "foo"]
+-- Processing input file 'foo' with verbose=false
+
+#eval main ["--verbose", "--input", "bar"]
+-- Processing input 'bar' with verbose=true
+
+/-!
+In this example `parseArguments` is actually three stacked monads, `StateM`, `ReaderM`, `Except`. Notice
+the convention of abbreviating long monadic types with an alias like `CliConfigM`.
+
+## Monad Lifting
+
+Lean makes it easy to compose functions that use different monads using a concept of automatic monad
+lifting.  You already used lifting in the above code, because you were able to compose
+`optionalSwitch` which has type `ReaderT Arguments (Except String) Bool` and call it from
+`parseArguments` which has a bigger type `StateT Config (ReaderT Arguments (Except String))`.
+This "just worked" because Lean did some magic with monad lifting.
+
+To give you a simpler example of this, suppose you have the following function:
+-/
+def divide (x : Float ) (y : Float): ExceptT String Id Float :=
+  if y == 0 then
+    throw "can't divide by zero"
+  else
+    pure (x / y)
+
+#eval divide 6 3 -- Except.ok 2.000000
+#eval divide 1 0 -- Except.error "can't divide by zero"
+/-!
+
+Notice here we used the `ExceptT` transformer, but we composed it with the `Id` identity monad.
+This is then the same as writing `Except String Float` since the identity monad does nothing.
+
+Now suppose you want to count the number of times divide is called and store the result in some
+global state:
+-/
+
+def divideCounter (x : Float) (y : Float) : StateT Nat (ExceptT String Id) Float := do
+  modify fun s => s + 1
+  divide x y
+
+#eval divideCounter 6 3 |>.run 0    -- Except.ok (2.000000, 1)
+#eval divideCounter 1 0 |>.run 0    -- Except.error "can't divide by zero"
+
+/-!
+
+The `modify` function is a helper which makes it easier to use `modifyGet` from the `StateM` monad.
+But something interesting is happening here, `divideCounter` is returning the value of
+`divide`, but the types don't match, yet it works?  This is monad lifting in action.
+
+You can see this more clearly with the following test:
+
+-/
+def liftTest (x : Except String Float) :
+  StateT Nat (Except String) Float := x
+
+#eval liftTest (divide 5 1) |>.run 3 -- Except.ok (5.000000, 3)
+
+/-!
+
+Notice that `liftTest` returned `x` without doing anything to it, yet that matched the return type
+`StateT Nat (Except String) Float`.  Monad lifting is provided by monad transformers.  if you
+`#print liftTest` you will see that Lean is implementing this using a call to a function named
+`monadLift` from the `MonadLift` type class:
+
+```lean,ignore
+class MonadLift (m : Type u → Type v) (n : Type u → Type w) where
+  monadLift : {α : Type u} → m α → n α
+```
+
+So `monadLift` is a function for lifting a computation from an inner `Monad m α ` to an outer `Monad n α`.
+You could replace `x` in `liftTest` with `monadLift x` if you want to be explicit about it.
+
+The StateT monad transformer defines an instance of `MonadLift` like this:
+
+```lean
+@[inline] protected def lift {α : Type u} (t : m α) : StateT σ m α :=
+  fun s => do let a ← t; pure (a, s)
+
+instance : MonadLift m (StateT σ m) := ⟨StateT.lift⟩
+```
+This means that any monad `m` can be wrapped in a `StateT` monad by using the function
+`fun s => do let a ← t; pure (a, s)` that takes state `s`, runs the inner monad action `t`, and
+returns the result and the new state in a pair `(a, s)` without making any changes to `s`.
+
+Because `MonadLift` is a type class, Lean can automatically find the required `monadLift`
+instances in order to make your code compile and in this way it was able to find the `StateT.lift`
+function and use it to wrap the result of `divide` so that the correct type is returned from
+`divideCounter`.
+
+If you have an instance `MonadLift m n` that means there is a way to turn a computation that happens
+inside of `m` into one that happens inside of `n` and (this is the key part) usually *without* the
+instance itself creating any additional data that feeds into the computation. This means you can in
+principle declare lifting instances from any monad to any other monad, it does not, however, mean
+that you should do this in all cases.  You can get a very nice report on how all this was done by
+adding the line `set_option trace.Meta.synthInstance true in` before `divideCounter` and moving you
+cursor to the end of the first line after `do`.
+
+This was a lot of detail, but it is very important to understand how monad lifting works because it
+is used heavily in Lean programs.
+
+## Transitive lifting
+
+There is also a transitive version of `MonadLift` called `MonadLiftT` which can lift multiple
+monad layers at once.  In the following example we added another monad layer with
+`ReaderT String ...` and notice that `x` is also automatically lifted to match.
+
+-/
+def liftTest2 (x : Except String Float) :
+  ReaderT String (StateT Nat (Except String)) Float := x
+
+#eval liftTest2 (divide 5 1) |>.run "" |>.run 3
+-- Except.ok (5.000000, 3)
+
+/-!
+
+The ReaderT monadLift is even simpler than the one for StateT:
+
+```lean,ignore
+instance  : MonadLift m (ReaderT ρ m) where
+  monadLift x := fun _ => x
+```
+
+This lift operation creates a function that defines the required `ReaderT` input
+argument, but the inner monad doesn't know or care about `ReaderT` so the
+monadLift function throws it away with the `_` then calls the inner monad action `x`.
+This is a perfectly legal implementation of the `ReaderM` monad.
+
+## Add your own Custom MonadLift
+
+This does not compile:
+-/
+def main2 : IO Unit := do
+  try
+    let ret ← divideCounter 5 2 |>.run 0
+    IO.println (toString ret)
+  catch e =>
+    IO.println e
+
+/-!
+saying:
+```
+typeclass instance problem is stuck, it is often due to metavariables
+  ToString ?m.4786
+```
+
+The reason is `divideCounter` returns the big `StateT Nat (ExceptT String Id) Float` and that type
+cannot be automatically lifted into the `main` return type of `IO Unit` unless you give it some
+help.
+
+The following custom `MonadLift` solves this problem:
+
+-/
+def liftIO (t : ExceptT String Id α) : IO α := do
+  match t with
+  | .ok r => EStateM.Result.ok r
+  | .error s => EStateM.Result.error s
+
+instance : MonadLift (ExceptT String Id) IO where
+  monadLift := liftIO
+
+def main3 : IO Unit := do
+  try
+    let ret ← divideCounter 5 2 |>.run 0
+    IO.println (toString ret)
+  catch e =>
+    IO.println e
+
+#eval main3 -- (2.500000, 1)
+/-!
+
+It turns out that the `IO` monad you see in your `main` function is based on the `EStateM.Result` type
+which is similar to the `Except` type but it has an additional return value. The `liftIO` function
+converts any `Except String α` into `IO α` by simply mapping the ok case of the `Except` to the
+`Result.ok` and the error case to the `Result.error`.
+
+## Lifting ExceptT
+
+In the previous [Except](except.lean.md) section you saw functions that `throw` Except
+values. When you get all the way back up to your `main` function which has type `IO Unit` you have
+the same problem you had above, because `Except String Float` doesn't match even if you use a
+`try/catch`.
+
+-/
+
+def main4 : IO Unit := do
+  try
+    let ret ← divide 5 0
+    IO.println (toString ret)  -- lifting happens here.
+  catch e =>
+    IO.println s!"Unhandled exception: {e}"
+
+#eval main4 -- Unhandled exception: can't divide by zero
+
+/-!
+
+Without the `liftIO` the `(toString ret)` expression would not compile with a similar error:
+
+```
+typeclass instance problem is stuck, it is often due to metavariables
+  ToString ?m.6007
+```
+
+So the general lesson is that if you see an error like this when using monads, check for
+a missing `MonadLift`.
+
+## Summary
+
+Now that you know how to combine your monads together, you're almost done with understanding the key
+concepts of monads! You could probably go out now and start writing some pretty nice code! But to
+truly master monads, you should know how to make your own, and there's one final concept that you
+should understand for that. This is the idea of type "laws". Each of the structures you've learned
+so far has a series of laws associated with it. And for your instances of these classes to make
+sense, they should follow the laws! Check out [Monad Laws](laws.lean.md).
+-/

doc/namespaces.md

@@ -0,0 +1,108 @@
+# Namespaces
+
+Lean provides us with the ability to group definitions into nested, hierarchical *namespaces*:
+
+```lean
+namespace Foo
+  def a : Nat := 5
+  def f (x : Nat) : Nat := x + 7
+
+  def fa : Nat := f a
+  def ffa : Nat := f (f a)
+
+  #check a
+  #check f
+  #check fa
+  #check ffa
+  #check Foo.fa
+end Foo
+
+-- #check a  -- error
+-- #check f  -- error
+#check Foo.a
+#check Foo.f
+#check Foo.fa
+#check Foo.ffa
+
+open Foo
+
+#check a
+#check f
+#check fa
+#check Foo.fa
+```
+
+When we declare that we are working in the namespace ``Foo``, every identifier we declare has
+a full name with prefix "``Foo.``" Within the namespace, we can refer to identifiers
+by their shorter names, but once we end the namespace, we have to use the longer names.
+
+The ``open`` command brings the shorter names into the current context. Often, when we import a
+module, we will want to open one or more of the namespaces it contains, to have access to the short identifiers.
+But sometimes we will want to leave this information hidden, for example, when they conflict with
+identifiers in another namespace we want to use. Thus namespaces give us a way to manage our working environment.
+
+For example, Lean groups definitions and theorems involving lists into a namespace ``List``.
+```lean
+#check List.nil
+#check List.cons
+#check List.map
+```
+We will discuss their types, below. The command ``open List`` allows us to use the shorter names:
+```lean
+open List
+
+#check nil
+#check cons
+#check map
+```
+Like sections, namespaces can be nested:
+```lean
+namespace Foo
+  def a : Nat := 5
+  def f (x : Nat) : Nat := x + 7
+
+  def fa : Nat := f a
+
+  namespace Bar
+    def ffa : Nat := f (f a)
+
+    #check fa
+    #check ffa
+  end Bar
+
+  #check fa
+  #check Bar.ffa
+end Foo
+
+#check Foo.fa
+#check Foo.Bar.ffa
+
+open Foo
+
+#check fa
+#check Bar.ffa
+```
+Namespaces that have been closed can later be reopened, even in another file:
+```lean
+namespace Foo
+  def a : Nat := 5
+  def f (x : Nat) : Nat := x + 7
+
+  def fa : Nat := f a
+end Foo
+
+#check Foo.a
+#check Foo.f
+
+namespace Foo
+  def ffa : Nat := f (f a)
+end Foo
+```
+
+Like sections, nested namespaces have to be closed in the order they are opened.
+Namespaces and sections serve different purposes: namespaces organize data and sections declare variables for insertion in definitions.
+Sections are also useful for delimiting the scope of commands such as ``set_option`` and ``open``.
+
+In many respects, however, a ``namespace ... end`` block behaves the same as a ``section ... end`` block.
+In particular, if you use the ``variable`` command within a namespace, its scope is limited to the namespace.
+Similarly, if you use an ``open`` command within a namespace, its effects disappear when the namespace is closed.

doc/nat.md

@@ -0,0 +1,68 @@
+# Natural numbers
+
+The `Nat` type represents the natural numbers, i.e., arbitrary-precision unsigned integers.
+There are no overflows.
+
+```lean
+#eval 100000000000000000 * 200000000000000000000 * 1000000000000000000000
+```
+
+A numeral is considered to be a `Nat` if there are no typing constraints.
+```lean
+#check 10    -- Nat
+#check id 10 -- Nat
+
+def f (x : Int) : Int :=
+  x - 1
+
+#eval f (3 - 5) -- 3 and 5 are `Int` since `f` expects an `Int`.
+-- -3
+```
+
+The operator `-` for `Nat` implements truncated subtraction.
+```lean
+#eval 10 - 5 -- 5
+#eval 5 - 10 -- 0
+
+theorem ex : 5 - 10 = 0 :=
+  rfl
+
+#eval (5:Int) - 10 -- -5
+```
+
+The operator `/` for `Nat` implements Euclidean division.
+```lean
+#eval 10 / 4 -- 2
+
+#check 10.0 / 4.0 -- Float
+#eval 10.0 / 4.0  -- 2.5
+```
+
+As we described in the previous sections, we define the `Nat` type as an `inductive` datatype.
+```lean
+# namespace hidden
+inductive Nat where
+  | zero : Nat
+  | succ : Nat → Nat
+# end hidden
+```
+However, the internal representation of `Nat` is optimized. Small natural numbers (i.e., < `2^63` in a 64-bit machine) are
+represented by a single machine word. Big numbers are implemented using [GMP](https://gmplib.org/manual/) numbers.
+We recommend you use fixed precision numeric types only in performance critical code.
+
+The Lean kernel has builtin support for the `Nat` type too, and can efficiently reduce `Nat` expressions during type checking.
+```lean
+#reduce 100000000000000000 * 200000000000000000000 * 1000000000000000000000
+
+theorem ex
+      : 1000000000000000 * 2000000000000000000 = 2000000000000000000000000000000000 :=
+  rfl
+```
+The sharp-eyed reader will notice that GMP is part of the Lean kernel trusted code base.
+We believe this is not a problem because you can use external type checkers to double-check your developments,
+and we consider GMP very trustworthy.
+Existing external type checkers for Lean 3 (e.g., [Trepplein](https://github.com/gebner/trepplein) and [TC](https://github.com/leanprover/tc))
+can be easily adapted to Lean 4.
+If you are still concerned after checking your development with multiple different external checkers because
+they may all rely on buggy arbitrary-precision libraries,
+you can develop your own certified arbitrary-precision library and use it to implement your own type checker for Lean.

doc/notation.md

@@ -0,0 +1,78 @@
+# Notations and Precedence
+
+The most basic syntax extension commands allow introducing new (or
+overloading existing) prefix, infix, and postfix operators.
+
+```lean
+infixl:65   " + " => HAdd.hAdd  -- left-associative
+infix:50    " = " => Eq         -- non-associative
+infixr:80   " ^ " => HPow.hPow  -- right-associative
+prefix:75  "-"   => Neg.neg
+# set_option quotPrecheck false
+postfix:max "⁻¹"  => Inv.inv
+```
+
+After the initial command name describing the operator kind (its
+"fixity"), we give the *parsing precedence* of the operator preceded
+by a colon `:`, then a new or existing token surrounded by double
+quotes (the whitespace is used for pretty printing), then the function
+this operator should be translated to after the arrow `=>`.
+
+The precedence is a natural number describing how "tightly" an
+operator binds to its arguments, encoding the order of operations. We
+can make this more precise by looking at what the commands above unfold to:
+
+```lean
+notation:65 lhs:65 " + " rhs:66 => HAdd.hAdd lhs rhs
+notation:50 lhs:51 " = " rhs:51 => Eq lhs rhs
+notation:80 lhs:81 " ^ " rhs:80 => HPow.hPow lhs rhs
+notation:75 "-" arg:75 => Neg.neg arg
+# set_option quotPrecheck false
+notation:1024 arg:1024 "⁻¹" => Inv.inv arg  -- `max` is a shorthand for precedence 1024
+```
+
+It turns out that all commands from the first code block are in fact
+command *macros* translating to the more general `notation` command.
+We will learn about writing such macros below. Instead of a single
+token, the `notation` command accepts a mixed sequence of tokens and
+named term placeholders with precedences, which can be referenced on
+the right-hand side of `=>` and will be replaced by the respective
+term parsed at that position. A placeholder with precedence `p`
+accepts only notations with precedence at least `p` in that place.
+Thus the string `a + b + c` cannot be parsed as the equivalent of `a +
+(b + c)` because the right-hand side operand of an `infixl` notation
+has precedence one greater than the notation itself. In contrast,
+`infixr` reuses the notation's precedence for the right-hand side
+operand, so `a ^ b ^ c` *can* be parsed as `a ^ (b ^ c)`. Note that if
+we used `notation` directly to introduce an infix notation like
+```lean
+# set_option quotPrecheck false
+notation:65 lhs:65 " ~ " rhs:65 => wobble lhs rhs
+```
+where the precedences do not sufficiently determine associativity,
+Lean's parser will default to right associativity. More precisely,
+Lean's parser follows a local *longest parse* rule in the presence of
+ambiguous grammars: when parsing the right-hand side of `a ~` in `a ~
+b ~ c`, it will continue parsing as long as possible (as the current
+precedence allows), not stopping after `b` but parsing `~ c` as well.
+Thus the term is equivalent to `a ~ (b ~ c)`.
+
+As mentioned above, the `notation` command allows us to define
+arbitrary *mixfix* syntax freely mixing tokens and placeholders.
+```lean
+# set_option quotPrecheck false
+notation:max "(" e ")" => e
+notation:10 Γ " ⊢ " e " : " τ => Typing Γ e τ
+```
+Placeholders without precedence default to `0`, i.e. they accept
+notations of any precedence in their place. If two notations overlap,
+we again apply the longest parse rule:
+```lean
+notation:65 a " + " b:66 " + " c:66 => a + b - c
+#eval 1 + 2 + 3  -- 0
+```
+The new notation is preferred to the binary notation since the latter,
+before chaining, would stop parsing after `1 + 2`. If there are
+multiple notations accepting the same longest parse, the choice will
+be delayed until elaboration, which will fail unless exactly one
+overload is type correct.

doc/option.md

@@ -0,0 +1 @@
+# Option

doc/organization.md

@@ -0,0 +1,4 @@
+# Organizational features
+
+In this section we introduce some organizational features of Lean that are not a part of its kernel per se,
+but make it possible to work in the framework more efficiently.

doc/reference.md

@@ -1,3 +0,0 @@
-# The Lean Reference Manual
-
-The latest version of the Lean reference manual is available [here](https://lean-lang.org/doc/reference/latest).

doc/sections.md

@@ -0,0 +1,70 @@
+# Variables and Sections
+
+Consider the following three function definitions:
+```lean
+def compose (α β γ : Type) (g : β → γ) (f : α → β) (x : α) : γ :=
+  g (f x)
+
+def doTwice (α : Type) (h : α → α) (x : α) : α :=
+  h (h x)
+
+def doThrice (α : Type) (h : α → α) (x : α) : α :=
+  h (h (h x))
+```
+
+Lean provides us with the ``variable`` command to make such declarations look more compact:
+
+```lean
+variable (α β γ : Type)
+
+def compose (g : β → γ) (f : α → β) (x : α) : γ :=
+  g (f x)
+
+def doTwice (h : α → α) (x : α) : α :=
+  h (h x)
+
+def doThrice (h : α → α) (x : α) : α :=
+  h (h (h x))
+```
+We can declare variables of any type, not just ``Type`` itself:
+```lean
+variable (α β γ : Type)
+variable (g : β → γ) (f : α → β) (h : α → α)
+variable (x : α)
+
+def compose := g (f x)
+def doTwice := h (h x)
+def doThrice := h (h (h x))
+
+#print compose
+#print doTwice
+#print doThrice
+```
+Printing them out shows that all three groups of definitions have exactly the same effect.
+
+The ``variable`` command instructs Lean to insert the declared variables as bound variables in definitions that refer to them.
+Lean is smart enough to figure out which variables are used explicitly or implicitly in a definition. We can therefore proceed as
+though ``α``, ``β``, ``γ``, ``g``, ``f``, ``h``, and ``x`` are fixed objects when we write our definitions, and let Lean abstract
+the definitions for us automatically.
+
+When declared in this way, a variable stays in scope until the end of the file we are working on.
+Sometimes, however, it is useful to limit the scope of a variable. For that purpose, Lean provides the notion of a ``section``:
+
+```lean
+section useful
+  variable (α β γ : Type)
+  variable (g : β → γ) (f : α → β) (h : α → α)
+  variable (x : α)
+
+  def compose := g (f x)
+  def doTwice := h (h x)
+  def doThrice := h (h (h x))
+end useful
+```
+
+When the section is closed, the variables go out of scope, and become nothing more than a distant memory.
+
+You do not have to indent the lines within a section. Nor do you have to name a section, which is to say,
+you can use an anonymous ``section`` / ``end`` pair.
+If you do name a section, however, you have to close it using the same name.
+Sections can also be nested, which allows you to declare new variables incrementally.

doc/string.md

@@ -0,0 +1 @@
+# Strings

doc/stringinterp.md

@@ -0,0 +1,58 @@
+# String interpolation
+
+The `s!` prefix identifies a string literal as an interpolated string.
+An interpolated string is a string literal that might contain interpolation expressions.
+When an interpolated string is resolved to a result string, items with interpolation expressions are
+replaced by the string representations of the expression results. The polymorphic method `toString` is used
+to convert the value into a string.
+
+String interpolation provides a more readable and convenient syntax to create formatted strings than
+a string composite formatting feature. The following example uses both features to produce the same output:
+
+```lean
+def name := "John"
+def age  := 28
+
+#eval IO.println s!"Hello, {name}! Are you {age} years old?"
+
+#eval IO.println ("Hello, " ++ name ++ "! Are you " ++ toString age ++ " years old?")
+
+-- `println! <interpolated-string>` is a macro for `IO.println s!<interpolated-string>`
+#eval println! "Hello, {name}! Are you {age} years old?"
+```
+
+# Structure of an interpolated string
+
+To identify a string literal as an interpolated string, prepend it with `s!`.
+Terms inside braces `{}` are ordinary expressions whose type implements the type class `ToString`.
+To include a curly brace `{` in your interpolated string, you must escape it using `\{`.
+You can nest interpolated strings inside interpolated strings.
+
+```lean
+def vals := [1, 2, 3]
+
+#eval IO.println s!"\{ vals := {vals} }"
+
+#eval IO.println s!"variables: {vals.map (fun i => s!"x_{i}")}"
+```
+
+# `ToString` instances
+
+You can define a `ToString` instance for your own datatypes.
+
+```lean
+structure Person where
+  name : String
+  age  : Nat
+
+instance : ToString Person where
+  toString : Person -> String
+    | { name := n, age := v } => s!"\{ name := {n}, age := {v} }"
+
+def person1 : Person := {
+  name := "John"
+  age  := 28
+}
+
+#eval println! "person1: {person1}"
+```

doc/struct.md

@@ -0,0 +1,227 @@
+# Structures
+
+Structure is a special case of inductive datatype. It has only one constructor and is not recursive.
+Similar to the `inductive` command, the `structure` command introduces a namespace with the same name.
+The general form is as follows:
+```
+structure <name> <parameters> <parent-structures> where
+  <constructor-name> :: <fields>
+```
+Most parts are optional. Here is our first example.
+```lean
+structure Point (α : Type u) where
+  x : α
+  y : α
+```
+In the example above, the constructor name is not provided. So, the constructor is named `mk` by Lean.
+Values of type ``Point`` are created using `Point.mk a b` or `{ x := a, y := b : Point α }`. The latter can be
+written as `{ x := a, y := b }` when the expected type is known.
+The fields of a point ``p`` are accessed using ``Point.x p`` and ``Point.y p``. You can also the more compact notation `p.x` and `p.y` as a shorthand
+for `Point.x p` and `Point.y p`.
+```lean
+# structure Point (α : Type u) where
+#  x : α
+#  y : α
+#check Point
+#check Point       -- Type u -> Type u
+#check @Point.mk   -- {α : Type u} → α → α → Point α
+#check @Point.x    -- {α : Type u} → Point α → α
+#check @Point.y    -- {α : Type u} → Point α → α
+
+#check Point.mk 10 20                   -- Point Nat
+#check { x := 10, y := 20 : Point Nat } -- Point Nat
+
+def mkPoint (a : Nat) : Point Nat :=
+  { x := a, y := a }
+
+#eval (Point.mk 10 20).x                   -- 10
+#eval (Point.mk 10 20).y                   -- 20
+#eval { x := 10, y := 20 : Point Nat }.x   -- 10
+#eval { x := 10, y := 20 : Point Nat }.y   -- 20
+
+def addXY (p : Point Nat) : Nat :=
+  p.x + p.y
+
+#eval addXY { x := 10, y := 20 }    -- 30
+```
+In the notation `{ ... }`, if the fields are in different lines, the `,` is optional.
+```lean
+# structure Point (α : Type u) where
+#  x : α
+#  y : α
+def mkPoint (a : Nat) : Point Nat := {
+  x := a
+  y := a
+}
+```
+You can also use `where` instead of `:= { ... }`.
+```lean
+# structure Point (α : Type u) where
+#  x : α
+#  y : α
+def mkPoint (a : Nat) : Point Nat where
+  x := a
+  y := a
+```
+
+Here are some simple theorems about our `Point` type.
+```lean
+# structure Point (α : Type u) where
+#  x : α
+#  y : α
+theorem ex1 (a b : α) : (Point.mk a b).x = a :=
+  rfl
+
+theorem ex2 (a b : α) : (Point.mk a b).y = b :=
+  rfl
+
+theorem ex3 (a b : α) : Point.mk a b = { x := a, y := b } :=
+  rfl
+```
+
+The dot notation is convenient not just for accessing the projections of a structure,
+but also for applying functions defined in a namespace with the same name.
+If ``p`` has type ``Point``, the expression ``p.foo`` is interpreted as ``Point.foo p``,
+assuming that the first argument to ``foo`` has type ``Point``.
+The expression ``p.add q`` is therefore shorthand for ``Point.add p q`` in the example below.
+```lean
+structure Point (α : Type u) where
+  x : α
+  y : α
+
+def Point.add (p q : Point Nat) : Point Nat :=
+  { x := p.x + q.x, y := p.y + q.y }
+
+def p : Point Nat := Point.mk 1 2
+def q : Point Nat := Point.mk 3 4
+
+#eval (p.add q).x  -- 4
+#eval (p.add q).y  -- 6
+```
+
+After we introduce type classes, we show how to define a function like ``add`` so that
+it works generically for elements of ``Point α`` rather than just ``Point Nat``,
+assuming ``α`` has an associated addition operation.
+
+More generally, given an expression ``p.foo x y z``, Lean will insert ``p`` at the first argument to ``foo`` of type ``Point``.
+For example, with the definition of scalar multiplication below, ``p.smul 3`` is interpreted as ``Point.smul 3 p``.
+
+```lean
+structure Point (α : Type u) where
+  x : α
+  y : α
+
+def Point.smul (n : Nat) (p : Point Nat) :=
+  Point.mk (n * p.x) (n * p.y)
+
+def p : Point Nat :=
+  Point.mk 1 2
+
+#eval (p.smul 3).x -- 3
+#eval (p.smul 3).y -- 6
+```
+
+##  Inheritance
+
+We can *extend* existing structures by adding new fields. This feature allows us to simulate a form of *inheritance*.
+
+```lean
+structure Point (α : Type u) where
+  x : α
+  y : α
+
+inductive Color where
+  | red
+  | green
+  | blue
+
+structure ColorPoint (α : Type u) extends Point α where
+  color : Color
+
+#check { x := 10, y := 20, color := Color.red : ColorPoint Nat }
+-- { toPoint := { x := 10, y := 20 }, color := Color.red }
+```
+The output for the `check` command above suggests how Lean encoded inheritance and multiple inheritance.
+Lean uses fields to each parent structure.
+
+```lean
+structure Foo where
+  x : Nat
+  y : Nat
+
+structure Boo where
+  w : Nat
+  z : Nat
+
+structure Bla extends Foo, Boo where
+  bit : Bool
+
+#check Bla.mk -- Foo → Boo → Bool → Bla
+#check Bla.mk { x := 10, y := 20 } { w := 30, z := 40 } true
+#check { x := 10, y := 20, w := 30, z := 40, bit := true : Bla }
+#check { toFoo := { x := 10, y := 20 },
+         toBoo := { w := 30, z := 40 },
+         bit := true : Bla }
+
+theorem ex :
+    Bla.mk { x := x, y := y } { w := w, z := z } b
+    =
+    { x := x, y := y, w := w, z := z, bit := b } :=
+  rfl
+```
+
+## Default field values
+
+You can assign default value to fields when declaring a new structure.
+```lean
+inductive MessageSeverity
+  | error | warning
+
+structure Message where
+  fileName : String
+  pos      : Option Nat      := none
+  severity : MessageSeverity := MessageSeverity.error
+  caption  : String          := ""
+  data     : String
+
+def msg1 : Message :=
+  { fileName := "foo.lean", data := "failed to import file" }
+
+#eval msg1.pos      -- none
+#eval msg1.fileName -- "foo.lean"
+#eval msg1.caption  -- ""
+```
+When extending a structure, you can not only add new fields, but provide new default values for existing fields.
+```lean
+# inductive MessageSeverity
+#  | error | warning
+# structure Message where
+#  fileName : String
+#  pos      : Option Nat      := none
+#  severity : MessageSeverity := MessageSeverity.error
+#  caption  : String          := ""
+#  data     : String
+structure MessageExt extends Message where
+  timestamp : Nat
+  caption   := "extended" -- new default value for field `caption`
+
+def msg2 : MessageExt where
+  fileName  := "bar.lean"
+  data      := "error at initialization"
+  timestamp := 10
+
+#eval msg2.fileName  -- "bar.lean"
+#eval msg2.timestamp -- 10
+#eval msg2.caption   -- "extended"
+```
+
+## Updating structure fields
+
+Structure fields can be updated using `{ <struct-val> with <field> := <new-value>, ... }`:
+
+```lean
+# structure Point (α : Type u) where
+#  x : α
+#  y : α
+def incrementX (p : Point Nat) : Point Nat := { p with x := p.x + 1 }
+```

doc/syntax.md

@@ -0,0 +1,20 @@
+# Syntax Extensions
+
+Lean's syntax can be extended and customized
+by users at every level, ranging from [basic "mixfix" notations](./notation.md)
+over [macro transformers](./macro_overview.md) to
+[type-aware elaborators](./elaborators.md). In fact, all builtin syntax is parsed and
+processed using the same mechanisms and APIs open to users. In this
+section, we will describe and explain the various extension points.
+Significant syntax extensions already builtin into Lean such as the
+[`do` notation](./do.md) are described in subsections.
+
+While introducing new syntax is a relatively rare feature in
+programming languages and sometimes even frowned upon because of its
+potential to obscure code, it is an invaluable tool in formalization
+for expressing established conventions and notations of the respective
+field succinctly in code. Going beyond basic notations, Lean's ability
+to factor out common boilerplate code into (well-behaved) macros and
+to embed entire custom domain specific languages (DSLs) to textually
+encode subproblems efficiently and readably can be of great benefit to
+both programmers and proof engineers alike.

doc/task.md

@@ -0,0 +1 @@
+# Task

doc/thunk.md

@@ -0,0 +1,104 @@
+# Thunks, Tasks, and Threads
+
+A `Thunk` is defined as
+```lean
+# namespace Ex
+# universe u
+structure Thunk (α : Type u) : Type u where
+  fn : Unit → α
+# end Ex
+```
+A `Thunk` encapsulates a computation without evaluation.
+That is, a `Thunk` stores the way of how the value would be computed.
+The Lean runtime has special support for `Thunk`s. It caches their values
+after they are computed for the first time. This feature is useful for implementing
+data structures such as lazy lists.
+Here is a small example using a `Thunk`.
+```lean
+def fib : Nat → Nat
+  | 0   => 0
+  | 1   => 1
+  | x+2 => fib (x+1) + fib x
+
+def f (c : Bool) (x : Thunk Nat) : Nat :=
+  if c then
+    x.get
+  else
+    0
+
+def g (c : Bool) (x : Nat) : Nat :=
+  f c (Thunk.mk (fun _ => fib x))
+
+#eval g false 1000
+```
+The function `f` above uses `x.get` to evaluate the `Thunk` `x`.
+The expression `Thunk.mk (fun _ => fib x)` creates a `Thunk` for computing `fib x`.
+Note that `fib` is a very naive function for computing the Fibonacci numbers,
+and it would an unreasonable amount of time to compute `fib 1000`. However, our
+test terminates instantaneously because the `Thunk` is not evaluated when `c` is `false`.
+Lean has a builtin coercion from any type `a` to `Thunk a`. You write the function `g` above as
+```lean
+# def fib : Nat → Nat
+#  | 0   => 0
+#  | 1   => 1
+#  | x+2 => fib (x+1) + fib x
+# def f (c : Bool) (x : Thunk Nat) : Nat :=
+#  if c then
+#    x.get
+#  else
+#    0
+def g (c : Bool) (x : Nat) : Nat :=
+  f c (fib x)
+
+#eval g false 1000
+```
+In the following example, we use the macro `dbg_trace` to demonstrate
+that the Lean runtime caches the value computed by a `Thunk`.
+We remark that the macro `dbg_trace` should be used for debugging purposes
+only.
+```lean
+def add1 (x : Nat) : Nat :=
+  dbg_trace "add1: {x}"
+  x + 1
+
+def double (x : Thunk Nat) : Nat :=
+  x.get + x.get
+
+def triple (x : Thunk Nat) : Nat :=
+  double x + x.get
+
+def test (x : Nat) : Nat :=
+  triple (add1 x)
+
+#eval test 2
+-- add1: 2
+-- 9
+```
+Note that the message `add1: 2` is printed only once.
+Now, consider the same example using `Unit -> Nat` instead of `Thunk Nat`.
+```lean
+def add1 (x : Nat) : Nat :=
+  dbg_trace "add1: {x}"
+  x + 1
+
+def double (x : Unit -> Nat) : Nat :=
+  x () + x ()
+
+def triple (x : Unit -> Nat) : Nat :=
+  double x + x ()
+
+def test (x : Nat) : Nat :=
+  triple (fun _ => add1 x)
+
+#eval test 2
+-- add1: 2
+-- add1: 2
+-- 9
+```
+Now, the message `add1: 2` is printed twice.
+It may come as a surprise that it was printed twice instead of three times.
+As we pointed out, `dbg_trace` is a macro used for debugging purposes only,
+and `add1` is still considered to be a pure function.
+The Lean compiler performs common subexpression elimination when compiling `double`,
+and the produced code for `double` executes `x ()` only once instead of twice.
+This transformation is safe because `x : Unit -> Nat` is pure.

doc/typeclass.md

@@ -0,0 +1,457 @@
+# Type classes
+
+Typeclasses were introduced as a principled way of enabling
+ad-hoc polymorphism in functional programming languages. We first observe that it
+would be easy to implement an ad-hoc polymorphic function (such as addition) if the
+function simply took the type-specific implementation of addition as an argument
+and then called that implementation on the remaining arguments. For example,
+suppose we declare a structure in Lean to hold implementations of addition
+```lean
+# namespace Ex
+structure Add (a : Type) where
+  add : a -> a -> a
+
+#check @Add.add
+-- Add.add : {a : Type} → Add a → a → a → a
+# end Ex
+```
+In the above Lean code, the field `add` has type
+`Add.add : {α : Type} → Add α → α → α → α`
+where the curly braces around the type `a` mean that it is an implicit argument.
+We could implement `double` by
+```lean
+# namespace Ex
+# structure Add (a : Type) where
+#  add : a -> a -> a
+def double (s : Add a) (x : a) : a :=
+  s.add x x
+
+#eval double { add := Nat.add } 10
+-- 20
+
+#eval double { add := Nat.mul } 10
+-- 100
+
+#eval double { add := Int.add } 10
+-- 20
+
+# end Ex
+```
+Note that you can double a natural number `n` by `double { add := Nat.add } n`.
+Of course, it would be highly cumbersome for users to manually pass the
+implementations around in this way.
+Indeed, it would defeat most of the potential benefits of ad-hoc
+polymorphism.
+
+The main idea behind typeclasses is to make arguments such as `Add a` implicit,
+and to use a database of user-defined instances to synthesize the desired instances
+automatically through a process known as typeclass resolution. In Lean, by changing
+`structure` to `class` in the example above, the type of `Add.add` becomes
+```lean
+# namespace Ex
+class Add (a : Type) where
+  add : a -> a -> a
+
+#check @Add.add
+-- Add.add : {a : Type} → [self : Add a] → a → a → a
+# end Ex
+```
+where the square brackets indicate that the argument of type `Add a` is *instance implicit*,
+i.e. that it should be synthesized using typeclass resolution. This version of
+`add` is the Lean analogue of the Haskell term `add :: Add a => a -> a -> a`.
+Similarly, we can register an instance by
+```lean
+# namespace Ex
+# class Add (a : Type) where
+#  add : a -> a -> a
+instance : Add Nat where
+  add := Nat.add
+
+# end Ex
+```
+Then for `n : Nat` and `m : Nat`, the term `Add.add n m` triggers typeclass resolution with the goal
+of `Add Nat`, and typeclass resolution will synthesize the instance above. In
+general, instances may depend on other instances in complicated ways. For example,
+you can declare an (anonymous) instance stating that if `a` has addition, then `Array a`
+has addition:
+```lean
+instance [Add a] : Add (Array a) where
+  add x y := Array.zipWith x y (· + ·)
+
+#eval Add.add #[1, 2] #[3, 4]
+-- #[4, 6]
+
+#eval #[1, 2] + #[3, 4]
+-- #[4, 6]
+```
+Note that `x + y` is notation for `Add.add x y` in Lean.
+
+The example above demonstrates how type classes are used to overload notation.
+Now, we explore another application. We often need an arbitrary element of a given type.
+Recall that types may not have any elements in Lean.
+It often happens that we would like a definition to return an arbitrary element in a "corner case."
+For example, we may like the expression ``head xs`` to be of type ``a`` when ``xs`` is of type ``List a``.
+Similarly, many theorems hold under the additional assumption that a type is not empty.
+For example, if ``a`` is a type, ``exists x : a, x = x`` is true only if ``a`` is not empty.
+The standard library defines a type class ``Inhabited`` to enable type class inference to infer a
+"default" or "arbitrary" element of an inhabited type.
+Let us start with the first step of the program above, declaring an appropriate class:
+
+```lean
+# namespace Ex
+class Inhabited (a : Sort u) where
+  default : a
+
+#check @Inhabited.default
+-- Inhabited.default : {a : Sort u} → [self : Inhabited a] → a
+# end Ex
+```
+Note `Inhabited.default` doesn't have any explicit argument.
+
+An element of the class ``Inhabited a`` is simply an expression of the form ``Inhabited.mk x``, for some element ``x : a``.
+The projection ``Inhabited.default`` will allow us to "extract" such an element of ``a`` from an element of ``Inhabited a``.
+Now we populate the class with some instances:
+
+```lean
+# namespace Ex
+# class Inhabited (a : Sort _) where
+#  default : a
+instance : Inhabited Bool where
+  default := true
+
+instance : Inhabited Nat where
+  default := 0
+
+instance : Inhabited Unit where
+  default := ()
+
+instance : Inhabited Prop where
+  default := True
+
+#eval (Inhabited.default : Nat)
+-- 0
+
+#eval (Inhabited.default : Bool)
+-- true
+# end Ex
+```
+You can use the command `export` to create the alias `default` for `Inhabited.default`
+```lean
+# namespace Ex
+# class Inhabited (a : Sort _) where
+#  default : a
+# instance : Inhabited Bool where
+#  default := true
+# instance : Inhabited Nat where
+#  default := 0
+# instance : Inhabited Unit where
+#  default := ()
+# instance : Inhabited Prop where
+#  default := True
+export Inhabited (default)
+
+#eval (default : Nat)
+-- 0
+
+#eval (default : Bool)
+-- true
+# end Ex
+```
+
+## Chaining Instances
+
+If that were the extent of type class inference, it would not be all that impressive;
+it would be simply a mechanism of storing a list of instances for the elaborator to find in a lookup table.
+What makes type class inference powerful is that one can *chain* instances. That is,
+an instance declaration can in turn depend on an implicit instance of a type class.
+This causes class inference to chain through instances recursively, backtracking when necessary, in a Prolog-like search.
+
+For example, the following definition shows that if two types ``a`` and ``b`` are inhabited, then so is their product:
+```lean
+instance [Inhabited a] [Inhabited b] : Inhabited (a × b) where
+  default := (default, default)
+```
+With this added to the earlier instance declarations, type class instance can infer, for example, a default element of ``Nat × Bool``:
+```lean
+# namespace Ex
+# class Inhabited (a : Sort u) where
+#  default : a
+# instance : Inhabited Bool where
+#  default := true
+# instance : Inhabited Nat where
+#  default := 0
+# opaque default [Inhabited a] : a :=
+#  Inhabited.default
+instance [Inhabited a] [Inhabited b] : Inhabited (a × b) where
+  default := (default, default)
+
+#eval (default : Nat × Bool)
+-- (0, true)
+# end Ex
+```
+Similarly, we can inhabit type function with suitable constant functions:
+```lean
+# namespace Ex
+# class Inhabited (a : Sort u) where
+#  default : a
+# opaque default [Inhabited a] : a :=
+#  Inhabited.default
+instance [Inhabited b] : Inhabited (a -> b) where
+  default := fun _ => default
+# end Ex
+```
+As an exercise, try defining default instances for other types, such as `List` and `Sum` types.
+
+The Lean standard library contains the definition `inferInstance`. It has type `{α : Sort u} → [i : α] → α`,
+and is useful for triggering the type class resolution procedure when the expected type is an instance.
+```lean
+#check (inferInstance : Inhabited Nat) -- Inhabited Nat
+
+def foo : Inhabited (Nat × Nat) :=
+  inferInstance
+
+theorem ex : foo.default = (default, default) :=
+  rfl
+```
+You can use the command `#print` to inspect how simple `inferInstance` is.
+```lean
+#print inferInstance
+```
+
+## ToString
+
+The polymorphic method `toString` has type `{α : Type u} → [ToString α] → α → String`. You implement the instance
+for your own types and use chaining to convert complex values into strings. Lean comes with `ToString` instances
+for most builtin types.
+```lean
+structure Person where
+  name : String
+  age  : Nat
+
+instance : ToString Person where
+  toString p := p.name ++ "@" ++ toString p.age
+
+#eval toString { name := "Leo", age := 542 : Person }
+#eval toString ({ name := "Daniel", age := 18 : Person }, "hello")
+```
+## Numerals
+
+Numerals are polymorphic in Lean. You can use a numeral (e.g., `2`) to denote an element of any type that implements
+the type class `OfNat`.
+```lean
+structure Rational where
+  num : Int
+  den : Nat
+  inv : den ≠ 0
+
+instance : OfNat Rational n where
+  ofNat := { num := n, den := 1, inv := by decide }
+
+instance : ToString Rational where
+  toString r := s!"{r.num}/{r.den}"
+
+#eval (2 : Rational) -- 2/1
+
+#check (2 : Rational) -- Rational
+#check (2 : Nat)      -- Nat
+```
+Lean elaborate the terms `(2 : Nat)` and `(2 : Rational)` as
+`OfNat.ofNat Nat 2 (instOfNatNat 2)` and
+`OfNat.ofNat Rational 2 (instOfNatRational 2)` respectively.
+We say the numerals `2` occurring in the elaborated terms are *raw* natural numbers.
+You can input the raw natural number `2` using the macro `nat_lit 2`.
+```lean
+#check nat_lit 2  -- Nat
+```
+Raw natural numbers are *not* polymorphic.
+
+The `OfNat` instance is parametric on the numeral. So, you can define instances for particular numerals.
+The second argument is often a variable as in the example above, or a *raw* natural number.
+```lean
+class Monoid (α : Type u) where
+  unit : α
+  op   : α → α → α
+
+instance [s : Monoid α] : OfNat α (nat_lit 1) where
+  ofNat := s.unit
+
+def getUnit [Monoid α] : α :=
+  1
+```
+
+Because many users were forgetting the `nat_lit` when defining `OfNat` instances, Lean also accepts `OfNat` instance
+declarations not using `nat_lit`. Thus, the following is also accepted.
+```lean
+class Monoid (α : Type u) where
+  unit : α
+  op   : α → α → α
+
+instance [s : Monoid α] : OfNat α 1 where
+  ofNat := s.unit
+
+def getUnit [Monoid α] : α :=
+  1
+```
+
+## Output parameters
+
+By default, Lean only tries to synthesize an instance `Inhabited T` when the term `T` is known and does not
+contain missing parts. The following command produces the error
+"failed to create type class instance for `Inhabited (Nat × ?m.1499)`" because the type has a missing part (i.e., the `_`).
+```lean
+# -- FIXME: should fail
+#check (inferInstance : Inhabited (Nat × _))
+```
+You can view the parameter of the type class `Inhabited` as an *input* value for the type class synthesizer.
+When a type class has multiple parameters, you can mark some of them as output parameters.
+Lean will start type class synthesizer even when these parameters have missing parts.
+In the following example, we use output parameters to define a *heterogeneous* polymorphic
+multiplication.
+```lean
+# namespace Ex
+class HMul (α : Type u) (β : Type v) (γ : outParam (Type w)) where
+  hMul : α → β → γ
+
+export HMul (hMul)
+
+instance : HMul Nat Nat Nat where
+  hMul := Nat.mul
+
+instance : HMul Nat (Array Nat) (Array Nat) where
+  hMul a bs := bs.map (fun b => hMul a b)
+
+#eval hMul 4 3           -- 12
+#eval hMul 4 #[2, 3, 4]  -- #[8, 12, 16]
+# end Ex
+```
+The parameters `α` and `β` are considered input parameters and `γ` an output one.
+Given an application `hMul a b`, after types of `a` and `b` are known, the type class
+synthesizer is invoked, and the resulting type is obtained from the output parameter `γ`.
+In the example above, we defined two instances. The first one is the homogeneous
+multiplication for natural numbers. The second is the scalar multiplication for arrays.
+Note that, you chain instances and generalize the second instance.
+```lean
+# namespace Ex
+class HMul (α : Type u) (β : Type v) (γ : outParam (Type w)) where
+  hMul : α → β → γ
+
+export HMul (hMul)
+
+instance : HMul Nat Nat Nat where
+  hMul := Nat.mul
+
+instance : HMul Int Int Int where
+  hMul := Int.mul
+
+instance [HMul α β γ] : HMul α (Array β) (Array γ) where
+  hMul a bs := bs.map (fun b => hMul a b)
+
+#eval hMul 4 3                    -- 12
+#eval hMul 4 #[2, 3, 4]           -- #[8, 12, 16]
+#eval hMul (-2) #[3, -1, 4]       -- #[-6, 2, -8]
+#eval hMul 2 #[#[2, 3], #[0, 4]]  -- #[#[4, 6], #[0, 8]]
+# end Ex
+```
+You can use our new scalar array multiplication instance on arrays of type `Array β`
+with a scalar of type `α` whenever you have an instance `HMul α β γ`.
+In the last `#eval`, note that the instance was used twice on an array of arrays.
+
+## Default instances
+
+In the class `HMul`, the parameters `α` and `β` are treated as input values.
+Thus, type class synthesis only starts after these two types are known. This may often
+be too restrictive.
+```lean
+# namespace Ex
+class HMul (α : Type u) (β : Type v) (γ : outParam (Type w)) where
+  hMul : α → β → γ
+
+export HMul (hMul)
+
+instance : HMul Int Int Int where
+  hMul := Int.mul
+
+def xs : List Int := [1, 2, 3]
+
+# -- TODO: fix error message
+-- Error "failed to create type class instance for HMul Int ?m.1767 (?m.1797 x)"
+-- #check fun y => xs.map (fun x => hMul x y)
+# end Ex
+```
+The instance `HMul` is not synthesized by Lean because the type of `y` has not been provided.
+However, it is natural to assume that the type of `y` and `x` should be the same in
+this kind of situation. We can achieve exactly that using *default instances*.
+```lean
+# namespace Ex
+class HMul (α : Type u) (β : Type v) (γ : outParam (Type w)) where
+  hMul : α → β → γ
+
+export HMul (hMul)
+
+@[default_instance]
+instance : HMul Int Int Int where
+  hMul := Int.mul
+
+def xs : List Int := [1, 2, 3]
+
+#check fun y => xs.map (fun x => hMul x y)  -- Int -> List Int
+# end Ex
+```
+By tagging the instance above with the attribute `default_instance`, we are instructing Lean
+to use this instance on pending type class synthesis problems.
+The actual Lean implementation defines homogeneous and heterogeneous classes for arithmetical operators.
+Moreover, `a+b`, `a*b`, `a-b`, `a/b`, and `a%b` are notations for the heterogeneous versions.
+The instance `OfNat Nat n` is the default instance (with priority `100`) for the `OfNat` class. This is why the numeral
+`2` has type `Nat` when the expected type is not known. You can define default instances with higher
+priority to override the builtin ones.
+```lean
+structure Rational where
+  num : Int
+  den : Nat
+  inv : den ≠ 0
+
+@[default_instance 200]
+instance : OfNat Rational n where
+  ofNat := { num := n, den := 1, inv := by decide }
+
+instance : ToString Rational where
+  toString r := s!"{r.num}/{r.den}"
+
+#check 2 -- Rational
+```
+Priorities are also useful to control the interaction between different default instances.
+For example, suppose `xs` has type `α`, when elaboration `xs.map (fun x => 2 * x)`, we want the homogeneous instance for multiplication
+to have higher priority than the default instance for `OfNat`. This is particularly important when we have implemented only the instance
+`HMul α α α`, and did not implement `HMul Nat α α`.
+Now, we reveal how the notation `a*b` is defined in Lean.
+```lean
+# namespace Ex
+class OfNat (α : Type u) (n : Nat) where
+  ofNat : α
+
+@[default_instance]
+instance (n : Nat) : OfNat Nat n where
+  ofNat := n
+
+class HMul (α : Type u) (β : Type v) (γ : outParam (Type w)) where
+  hMul : α → β → γ
+
+class Mul (α : Type u) where
+  mul : α → α → α
+
+@[default_instance 10]
+instance [Mul α] : HMul α α α where
+  hMul a b := Mul.mul a b
+
+infixl:70 " * "  => HMul.hMul
+# end Ex
+```
+The `Mul` class is convenient for types that only implement the homogeneous multiplication.
+
+## Scoped Instances
+
+TODO
+
+## Local Instances
+
+TODO

doc/uint.md

@@ -0,0 +1 @@
+# Fixed precision unsigned integers

doc/unifhint.md

@@ -0,0 +1,5 @@
+# Unification Hints
+
+
+
+TODO

script/release_notes.py

@@ -1,145 +0,0 @@
-#!/usr/bin/env python3
-
-import sys
-import re
-import json
-import requests
-import subprocess
-from collections import defaultdict
-from git import Repo
-
-def get_commits_since_tag(repo, tag):
-    try:
-        tag_commit = repo.commit(tag)
-        commits = list(repo.iter_commits(f"{tag_commit.hexsha}..HEAD"))
-        return [
-            (commit.hexsha, commit.message.splitlines()[0], commit.message)
-            for commit in commits
-        ]
-    except Exception as e:
-        sys.stderr.write(f"Error retrieving commits: {e}\n")
-        sys.exit(1)
-
-def check_pr_number(first_line):
-    match = re.search(r"\(\#(\d+)\)$", first_line)
-    if match:
-        return int(match.group(1))
-    return None
-
-def fetch_pr_labels(pr_number):
-    try:
-        # Use gh CLI to fetch PR details
-        result = subprocess.run([
-            "gh", "api", f"repos/leanprover/lean4/pulls/{pr_number}"
-        ], capture_output=True, text=True, check=True)
-        pr_data = result.stdout
-        pr_json = json.loads(pr_data)
-        return [label["name"] for label in pr_json.get("labels", [])]
-    except subprocess.CalledProcessError as e:
-        sys.stderr.write(f"Failed to fetch PR #{pr_number} using gh: {e.stderr}\n")
-        return []
-
-def format_section_title(label):
-    title = label.replace("changelog-", "").capitalize()
-    if title == "Doc":
-        return "Documentation"
-    elif title == "Pp":
-        return "Pretty Printing"
-    return title
-
-def sort_sections_order():
-    return [
-        "Language",
-        "Library",
-        "Compiler",
-        "Pretty Printing",
-        "Documentation",
-        "Server",
-        "Lake",
-        "Other",
-        "Uncategorised"
-    ]
-
-def format_markdown_description(pr_number, description):
-    link = f"[#{pr_number}](https://github.com/leanprover/lean4/pull/{pr_number})"
-    return f"{link} {description}"
-
-def main():
-    if len(sys.argv) != 2:
-        sys.stderr.write("Usage: script.py <git-tag>\n")
-        sys.exit(1)
-
-    tag = sys.argv[1]
-    try:
-        repo = Repo(".")
-    except Exception as e:
-        sys.stderr.write(f"Error opening Git repository: {e}\n")
-        sys.exit(1)
-
-    commits = get_commits_since_tag(repo, tag)
-
-    sys.stderr.write(f"Found {len(commits)} commits since tag {tag}:\n")
-    for commit_hash, first_line, _ in commits:
-        sys.stderr.write(f"- {commit_hash}: {first_line}\n")
-
-    changelog = defaultdict(list)
-
-    for commit_hash, first_line, full_message in commits:
-        # Skip commits with the specific first lines
-        if first_line == "chore: update stage0" or first_line.startswith("chore: CI: bump "):
-            continue
-
-        pr_number = check_pr_number(first_line)
-
-        if not pr_number:
-            sys.stderr.write(f"No PR number found in {first_line}\n")
-            continue
-
-        # Remove the first line from the full_message for further processing
-        body = full_message[len(first_line):].strip()
-
-        paragraphs = body.split('\n\n')
-        second_paragraph = paragraphs[0] if len(paragraphs) > 0 else ""
-
-        labels = fetch_pr_labels(pr_number)
-
-        # Skip entries with the "changelog-no" label
-        if "changelog-no" in labels:
-            continue
-
-        report_errors = first_line.startswith("feat:") or first_line.startswith("fix:")
-
-        if not second_paragraph.startswith("This PR "):
-            if report_errors:
-                sys.stderr.write(f"No PR description found in commit:\n{commit_hash}\n{first_line}\n{body}\n\n")
-                fallback_description = re.sub(r":$", "", first_line.split(" ", 1)[1]).rsplit(" (#", 1)[0]
-                markdown_description = format_markdown_description(pr_number, fallback_description)
-            else:
-                continue
-        else:
-            markdown_description = format_markdown_description(pr_number, second_paragraph.replace("This PR ", ""))
-
-        changelog_labels = [label for label in labels if label.startswith("changelog-")]
-        if len(changelog_labels) > 1:
-            sys.stderr.write(f"Warning: Multiple changelog-* labels found for PR #{pr_number}: {changelog_labels}\n")
-
-        if not changelog_labels:
-            if report_errors:
-                sys.stderr.write(f"Warning: No changelog-* label found for PR #{pr_number}\n")
-            else:
-                continue
-
-        for label in changelog_labels:
-            changelog[label].append((pr_number, markdown_description))
-
-    section_order = sort_sections_order()
-    sorted_changelog = sorted(changelog.items(), key=lambda item: section_order.index(format_section_title(item[0])) if format_section_title(item[0]) in section_order else len(section_order))
-
-    for label, entries in sorted_changelog:
-        section_title = format_section_title(label) if label != "Uncategorised" else "Uncategorised"
-        print(f"## {section_title}\n")
-        for _, entry in sorted(entries, key=lambda x: x[0]):
-            print(f"* {entry}\n")
-
-if __name__ == "__main__":
-    main()

src/Init/Core.lean

@@ -2116,14 +2116,16 @@ instance : Commutative Or := ⟨fun _ _ => propext or_comm⟩
 instance : Commutative And := ⟨fun _ _ => propext and_comm⟩
 instance : Commutative Iff := ⟨fun _ _ => propext iff_comm⟩
 
-/-- `Refl r` means the binary relation `r` is reflexive, that is, `r x x` always holds. -/
+/-- `IsRefl X r` means the binary relation `r` on `X` is reflexive. -/
 class Refl (r : α → α → Prop) : Prop where
   /-- A reflexive relation satisfies `r a a`. -/
   refl : ∀ a, r a a
 
-/-- `Antisymm r` says that `r` is antisymmetric, that is, `r a b → r b a → a = b`. -/
+/--
+`Antisymm (·≤·)` says that `(·≤·)` is antisymmetric, that is, `a ≤ b → b ≤ a → a = b`.
+-/
 class Antisymm (r : α → α → Prop) : Prop where
-  /-- An antisymmetric relation `r` satisfies `r a b → r b a → a = b`. -/
+  /-- An antisymmetric relation `(·≤·)` satisfies `a ≤ b → b ≤ a → a = b`. -/
   antisymm (a b : α) : r a b → r b a → a = b
 
 @[deprecated Antisymm (since := "2024-10-16"), inherit_doc Antisymm]
@@ -2141,8 +2143,8 @@ class Total (r : α → α → Prop) : Prop where
   /-- A total relation satisfies `r a b ∨ r b a`. -/
   total : ∀ a b, r a b ∨ r b a
 
-/-- `Irrefl r` means the binary relation `r` is irreflexive, that is, `r x x` never
-holds. -/
+/-- `Irrefl X r` means the binary relation `r` on `X` is irreflexive (that is, `r x x` never
+holds). -/
 class Irrefl (r : α → α → Prop) : Prop where
   /-- An irreflexive relation satisfies `¬ r a a`. -/
   irrefl : ∀ a, ¬r a a

src/Init/Data/Array/Lemmas.lean

@@ -27,7 +27,7 @@ namespace Array
 
 /-! ### toList -/
 
-@[simp] theorem toList_inj {a b : Array α} : a.toList = b.toList ↔ a = b := by
+theorem toList_inj {a b : Array α} : a.toList = b.toList ↔ a = b := by
   cases a; cases b; simp
 
 @[simp] theorem toList_eq_nil_iff (l : Array α) : l.toList = [] ↔ l = #[] := by

src/Init/Data/Array/Lex/Lemmas.lean

@@ -11,11 +11,8 @@ namespace Array
 
 /-! ### Lexicographic ordering -/
 
-@[simp] theorem _root_.List.lt_toArray [LT α] (l₁ l₂ : List α) : l₁.toArray < l₂.toArray ↔ l₁ < l₂ := Iff.rfl
-@[simp] theorem _root_.List.le_toArray [LT α] (l₁ l₂ : List α) : l₁.toArray ≤ l₂.toArray ↔ l₁ ≤ l₂ := Iff.rfl
-
-@[simp] theorem lt_toList [LT α] (l₁ l₂ : Array α) : l₁.toList < l₂.toList ↔ l₁ < l₂ := Iff.rfl
-@[simp] theorem le_toList [LT α] (l₁ l₂ : Array α) : l₁.toList ≤ l₂.toList ↔ l₁ ≤ l₂ := Iff.rfl
+@[simp] theorem lt_toArray [LT α] (l₁ l₂ : List α) : l₁.toArray < l₂.toArray ↔ l₁ < l₂ := Iff.rfl
+@[simp] theorem le_toArray [LT α] (l₁ l₂ : List α) : l₁.toArray ≤ l₂.toArray ↔ l₁ ≤ l₂ := Iff.rfl
 
 theorem not_lt_iff_ge [LT α] (l₁ l₂ : List α) : ¬ l₁ < l₂ ↔ l₂ ≤ l₁ := Iff.rfl
 theorem not_le_iff_gt [DecidableEq α] [LT α] [DecidableLT α] (l₁ l₂ : List α) :
@@ -62,7 +59,6 @@ protected theorem lt_irrefl [LT α] [Std.Irrefl (· < · : α → α → Prop)]
 instance ltIrrefl [LT α] [Std.Irrefl (· < · : α → α → Prop)] : Std.Irrefl (α := Array α) (· < ·) where
   irrefl := Array.lt_irrefl
 
-@[simp] theorem not_lt_empty [LT α] (l : Array α) : ¬ l < #[] := List.not_lt_nil l.toList
 @[simp] theorem empty_le [LT α] (l : Array α) : #[] ≤ l := List.nil_le l.toList
 
 @[simp] theorem le_empty [LT α] (l : Array α) : l ≤ #[] ↔ l = #[] := by
@@ -78,12 +74,13 @@ protected theorem le_refl [LT α] [i₀ : Std.Irrefl (· < · : α → α → Pr
 instance [LT α] [Std.Irrefl (· < · : α → α → Prop)] : Std.Refl (· ≤ · : Array α → Array α → Prop) where
   refl := Array.le_refl
 
-protected theorem lt_trans [LT α]
+protected theorem lt_trans [LT α] [DecidableLT α]
     [i₁ : Trans (· < · : α → α → Prop) (· < ·) (· < ·)]
     {l₁ l₂ l₃ : Array α} (h₁ : l₁ < l₂) (h₂ : l₂ < l₃) : l₁ < l₃ :=
   List.lt_trans h₁ h₂
 
-instance [LT α] [Trans (· < · : α → α → Prop) (· < ·) (· < ·)] :
+instance [LT α] [DecidableLT α]
+    [Trans (· < · : α → α → Prop) (· < ·) (· < ·)] :
     Trans (· < · : Array α → Array α → Prop) (· < ·) (· < ·) where
   trans h₁ h₂ := Array.lt_trans h₁ h₂
 
@@ -111,7 +108,7 @@ instance [DecidableEq α] [LT α] [DecidableLT α]
     Trans (· ≤ · : Array α → Array α → Prop) (· ≤ ·) (· ≤ ·) where
   trans h₁ h₂ := Array.le_trans h₁ h₂
 
-protected theorem lt_asymm [LT α]
+protected theorem lt_asymm [DecidableEq α] [LT α] [DecidableLT α]
     [i : Std.Asymm (· < · : α → α → Prop)]
     {l₁ l₂ : Array α} (h : l₁ < l₂) : ¬ l₂ < l₁ := List.lt_asymm h
 
@@ -121,31 +118,13 @@ instance [DecidableEq α] [LT α] [DecidableLT α]
   asymm _ _ := Array.lt_asymm
 
 protected theorem le_total [DecidableEq α] [LT α] [DecidableLT α]
-    [i : Std.Total (¬ · < · : α → α → Prop)] (l₁ l₂ : Array α) : l₁ ≤ l₂ ∨ l₂ ≤ l₁ :=
-  List.le_total _ _
-
-@[simp] protected theorem not_lt [LT α]
-    {l₁ l₂ : Array α} : ¬ l₁ < l₂ ↔ l₂ ≤ l₁ := Iff.rfl
-
-@[simp] protected theorem not_le [DecidableEq α] [LT α] [DecidableLT α]
-    {l₁ l₂ : Array α} : ¬ l₂ ≤ l₁ ↔ l₁ < l₂ := Decidable.not_not
-
-protected theorem le_of_lt [DecidableEq α] [LT α] [DecidableLT α]
-    [i : Std.Total (¬ · < · : α → α → Prop)]
-    {l₁ l₂ : Array α} (h : l₁ < l₂) : l₁ ≤ l₂ :=
-  List.le_of_lt h
-
-theorem le_iff_lt_or_eq [DecidableEq α] [LT α] [DecidableLT α]
-    [Std.Irrefl (· < · : α → α → Prop)]
-    [Std.Antisymm (¬ · < · : α → α → Prop)]
-    [Std.Total (¬ · < · : α → α → Prop)]
-    {l₁ l₂ : Array α} : l₁ ≤ l₂ ↔ l₁ < l₂ ∨ l₁ = l₂ := by
-  simpa using List.le_iff_lt_or_eq (l₁ := l₁.toList) (l₂ := l₂.toList)
+    [i : Std.Total (¬ · < · : α → α → Prop)] {l₁ l₂ : Array α} : l₁ ≤ l₂ ∨ l₂ ≤ l₁ :=
+  List.le_total
 
 instance [DecidableEq α] [LT α] [DecidableLT α]
     [Std.Total (¬ · < · : α → α → Prop)] :
     Std.Total (· ≤ · : Array α → Array α → Prop) where
-  total := Array.le_total
+  total _ _ := Array.le_total
 
 @[simp] theorem lex_eq_true_iff_lt [DecidableEq α] [LT α] [DecidableLT α]
     {l₁ l₂ : Array α} : lex l₁ l₂ = true ↔ l₁ < l₂ := by
@@ -234,48 +213,4 @@ theorem le_iff_exists [DecidableEq α] [LT α] [DecidableLT α]
   cases l₂
   simp [List.le_iff_exists]
 
-theorem append_left_lt [LT α] {l₁ l₂ l₃ : Array α} (h : l₂ < l₃) :
-    l₁ ++ l₂ < l₁ ++ l₃ := by
-  cases l₁
-  cases l₂
-  cases l₃
-  simpa using List.append_left_lt h
-
-theorem append_left_le [DecidableEq α] [LT α] [DecidableLT α]
-    [Std.Irrefl (· < · : α → α → Prop)]
-    [Std.Asymm (· < · : α → α → Prop)]
-    [Std.Antisymm (¬ · < · : α → α → Prop)]
-    {l₁ l₂ l₃ : Array α} (h : l₂ ≤ l₃) :
-    l₁ ++ l₂ ≤ l₁ ++ l₃ := by
-  cases l₁
-  cases l₂
-  cases l₃
-  simpa using List.append_left_le h
-
-theorem le_append_left [LT α] [Std.Irrefl (· < · : α → α → Prop)]
-    {l₁ l₂ : Array α} : l₁ ≤ l₁ ++ l₂ := by
-  cases l₁
-  cases l₂
-  simpa using List.le_append_left
-
-theorem map_lt [LT α] [LT β]
-    {l₁ l₂ : Array α} {f : α → β} (w : ∀ x y, x < y → f x < f y) (h : l₁ < l₂) :
-    map f l₁ < map f l₂ := by
-  cases l₁
-  cases l₂
-  simpa using List.map_lt w h
-
-theorem map_le [DecidableEq α] [LT α] [DecidableLT α] [DecidableEq β] [LT β] [DecidableLT β]
-    [Std.Irrefl (· < · : α → α → Prop)]
-    [Std.Asymm (· < · : α → α → Prop)]
-    [Std.Antisymm (¬ · < · : α → α → Prop)]
-    [Std.Irrefl (· < · : β → β → Prop)]
-    [Std.Asymm (· < · : β → β → Prop)]
-    [Std.Antisymm (¬ · < · : β → β → Prop)]
-    {l₁ l₂ : Array α} {f : α → β} (w : ∀ x y, x < y → f x < f y) (h : l₁ ≤ l₂) :
-    map f l₁ ≤ map f l₂ := by
-  cases l₁
-  cases l₂
-  simpa using List.map_le w h
-
 end Array

src/Init/Data/List/Lex.lean

@@ -5,7 +5,6 @@ Authors: Kim Morrison
 -/
 prelude
 import Init.Data.List.Lemmas
-import Init.Data.List.Nat.TakeDrop
 
 namespace List
 
@@ -34,7 +33,6 @@ instance ltIrrefl [LT α] [Std.Irrefl (· < · : α → α → Prop)] : Std.Irre
 
 @[simp] theorem not_lex_nil : ¬Lex r l [] := fun h => nomatch h
 
-@[simp] theorem not_lt_nil [LT α] (l : List α) : ¬ l < [] := fun h => nomatch h
 @[simp] theorem nil_le [LT α] (l : List α) : [] ≤ l := fun h => nomatch h
 
 @[simp] theorem not_nil_lex_iff : ¬Lex r [] l ↔ l = [] := by
@@ -61,10 +59,6 @@ theorem cons_lt_cons_iff [LT α] {a b} {l₁ l₂ : List α} :
   dsimp only [instLT, List.lt]
   simp [cons_lex_cons_iff]
 
-@[simp] theorem cons_lt_cons_self [LT α] [i₀ : Std.Irrefl (· < · : α → α → Prop)] {l₁ l₂ : List α} :
-    (a :: l₁) < (a :: l₂) ↔ l₁ < l₂ := by
-  simp [cons_lt_cons_iff, i₀.irrefl]
-
 theorem not_cons_lex_cons_iff [DecidableEq α] [DecidableRel r] {a b} {l₁ l₂ : List α} :
     ¬ Lex r (a :: l₁) (b :: l₂) ↔ (¬ r a b ∧ a ≠ b) ∨ (¬ r a b ∧ ¬ Lex r l₁ l₂) := by
   rw [cons_lex_cons_iff, not_or, Decidable.not_and_iff_or_not, and_or_left]
@@ -126,7 +120,7 @@ protected theorem le_refl [LT α] [i₀ : Std.Irrefl (· < · : α → α → Pr
 instance [LT α] [Std.Irrefl (· < · : α → α → Prop)] : Std.Refl (· ≤ · : List α → List α → Prop) where
   refl := List.le_refl
 
-theorem lex_trans {r : α → α → Prop}
+theorem lex_trans {r : α → α → Prop} [DecidableRel r]
     (lt_trans : ∀ {x y z : α}, r x y → r y z → r x z)
     (h₁ : Lex r l₁ l₂) (h₂ : Lex r l₂ l₃) : Lex r l₁ l₃ := by
   induction h₁ generalizing l₃ with
@@ -143,13 +137,14 @@ theorem lex_trans {r : α → α → Prop}
     | .cons ih =>
       exact List.Lex.cons (ih2 ih)
 
-protected theorem lt_trans [LT α]
+protected theorem lt_trans [LT α] [DecidableLT α]
     [i₁ : Trans (· < · : α → α → Prop) (· < ·) (· < ·)]
     {l₁ l₂ l₃ : List α} (h₁ : l₁ < l₂) (h₂ : l₂ < l₃) : l₁ < l₃ := by
   simp only [instLT, List.lt] at h₁ h₂ ⊢
   exact lex_trans (fun h₁ h₂ => i₁.trans h₁ h₂) h₁ h₂
 
-instance [LT α] [Trans (· < · : α → α → Prop) (· < ·) (· < ·)] :
+instance [LT α] [DecidableLT α]
+    [Trans (· < · : α → α → Prop) (· < ·) (· < ·)] :
     Trans (· < · : List α → List α → Prop) (· < ·) (· < ·) where
   trans h₁ h₂ := List.lt_trans h₁ h₂
 
@@ -202,7 +197,7 @@ instance [DecidableEq α] [LT α] [DecidableLT α]
     Trans (· ≤ · : List α → List α → Prop) (· ≤ ·) (· ≤ ·) where
   trans h₁ h₂ := List.le_trans h₁ h₂
 
-theorem lex_asymm {r : α → α → Prop}
+theorem lex_asymm {r : α → α → Prop} [DecidableRel r]
     (h : ∀ {x y : α}, r x y → ¬ r y x) : ∀ {l₁ l₂ : List α}, Lex r l₁ l₂ → ¬ Lex r l₂ l₁
   | nil, _, .nil => by simp
   | x :: l₁, y :: l₂, .rel h₁ =>
@@ -214,11 +209,12 @@ theorem lex_asymm {r : α → α → Prop}
     | .rel h₂ => h h₂ h₂
     | .cons h₂ => lex_asymm h h₁ h₂
 
-protected theorem lt_asymm [LT α]
+protected theorem lt_asymm [DecidableEq α] [LT α] [DecidableLT α]
     [i : Std.Asymm (· < · : α → α → Prop)]
     {l₁ l₂ : List α} (h : l₁ < l₂) : ¬ l₂ < l₁ := lex_asymm (i.asymm _ _) h
 
-instance [LT α] [Std.Asymm (· < · : α → α → Prop)] :
+instance [DecidableEq α] [LT α] [DecidableLT α]
+    [Std.Asymm (· < · : α → α → Prop)] :
     Std.Asymm (· < · : List α → List α → Prop) where
   asymm _ _ := List.lt_asymm
 
@@ -238,43 +234,13 @@ theorem not_lex_total [DecidableEq α] {r : α → α → Prop} [DecidableRel r]
     obtain (_ | _) := not_lex_total h l₁ l₂ <;> contradiction
 
 protected theorem le_total [DecidableEq α] [LT α] [DecidableLT α]
-    [i : Std.Total (¬ · < · : α → α → Prop)] (l₁ l₂ : List α) : l₁ ≤ l₂ ∨ l₂ ≤ l₁ :=
+    [i : Std.Total (¬ · < · : α → α → Prop)] {l₁ l₂ : List α} : l₁ ≤ l₂ ∨ l₂ ≤ l₁ :=
   not_lex_total i.total l₂ l₁
 
 instance [DecidableEq α] [LT α] [DecidableLT α]
     [Std.Total (¬ · < · : α → α → Prop)] :
     Std.Total (· ≤ · : List α → List α → Prop) where
-  total := List.le_total
-
-@[simp] protected theorem not_lt [LT α]
-    {l₁ l₂ : List α} : ¬ l₁ < l₂ ↔ l₂ ≤ l₁ := Iff.rfl
-
-@[simp] protected theorem not_le [DecidableEq α] [LT α] [DecidableLT α]
-    {l₁ l₂ : List α} : ¬ l₂ ≤ l₁ ↔ l₁ < l₂ := Decidable.not_not
-
-protected theorem le_of_lt [DecidableEq α] [LT α] [DecidableLT α]
-    [i : Std.Total (¬ · < · : α → α → Prop)]
-    {l₁ l₂ : List α} (h : l₁ < l₂) : l₁ ≤ l₂ := by
-  obtain (h' | h') := List.le_total l₁ l₂
-  · exact h'
-  · exfalso
-    exact h' h
-
-theorem le_iff_lt_or_eq [DecidableEq α] [LT α] [DecidableLT α]
-    [Std.Irrefl (· < · : α → α → Prop)]
-    [Std.Antisymm (¬ · < · : α → α → Prop)]
-    [Std.Total (¬ · < · : α → α → Prop)]
-    {l₁ l₂ : List α} : l₁ ≤ l₂ ↔ l₁ < l₂ ∨ l₁ = l₂ := by
-  constructor
-  · intro h
-    by_cases h' : l₂ ≤ l₁
-    · right
-      apply List.le_antisymm h h'
-    · left
-      exact Decidable.not_not.mp h'
-  · rintro (h | rfl)
-    · exact List.le_of_lt h
-    · exact List.le_refl l₁
+  total _ _ := List.le_total
 
 theorem lex_eq_decide_lex [DecidableEq α] (lt : α → α → Bool) :
     lex l₁ l₂ lt = decide (Lex (fun x y => lt x y) l₁ l₂) := by
@@ -461,63 +427,4 @@ theorem le_iff_exists [DecidableEq α] [LT α] [DecidableLT α]
   · simpa using Std.Asymm.asymm
   · simpa using Std.Antisymm.antisymm
 
-theorem append_left_lt [LT α] {l₁ l₂ l₃ : List α} (h : l₂ < l₃) :
-    l₁ ++ l₂ < l₁ ++ l₃ := by
-  induction l₁ with
-  | nil => simp [h]
-  | cons a l₁ ih => simp [cons_lt_cons_iff, ih]
-
-theorem append_left_le [DecidableEq α] [LT α] [DecidableLT α]
-    [Std.Irrefl (· < · : α → α → Prop)]
-    [Std.Asymm (· < · : α → α → Prop)]
-    [Std.Antisymm (¬ · < · : α → α → Prop)]
-    {l₁ l₂ l₃ : List α} (h : l₂ ≤ l₃) :
-    l₁ ++ l₂ ≤ l₁ ++ l₃ := by
-  induction l₁ with
-  | nil => simp [h]
-  | cons a l₁ ih => simp [cons_le_cons_iff, ih]
-
-theorem le_append_left [LT α] [Std.Irrefl (· < · : α → α → Prop)]
-    {l₁ l₂ : List α} : l₁ ≤ l₁ ++ l₂ := by
-  intro h
-  match l₁, h with
-  | nil, h => simp at h
-  | cons a l₁, h => exact le_append_left (by simpa using h)
-
-theorem IsPrefix.le [LT α] [Std.Irrefl (· < · : α → α → Prop)]
-    {l₁ l₂ : List α} (h : l₁ <+: l₂) : l₁ ≤ l₂ := by
-  rcases h with ⟨_, rfl⟩
-  apply le_append_left
-
-theorem map_lt [LT α] [LT β]
-    {l₁ l₂ : List α} {f : α → β} (w : ∀ x y, x < y → f x < f y) (h : l₁ < l₂) :
-    map f l₁ < map f l₂ := by
-  match l₁, l₂, h with
-  | nil, nil, h => simp at h
-  | nil, cons b l₂, h => simp
-  | cons a l₁, nil, h => simp at h
-  | cons a l₁, cons _ l₂, .cons h =>
-    simp [cons_lt_cons_iff, map_lt w (by simpa using h)]
-  | cons a l₁, cons b l₂, .rel h =>
-    simp [cons_lt_cons_iff, w, h]
-
-theorem map_le [DecidableEq α] [LT α] [DecidableLT α] [DecidableEq β] [LT β] [DecidableLT β]
-    [Std.Irrefl (· < · : α → α → Prop)]
-    [Std.Asymm (· < · : α → α → Prop)]
-    [Std.Antisymm (¬ · < · : α → α → Prop)]
-    [Std.Irrefl (· < · : β → β → Prop)]
-    [Std.Asymm (· < · : β → β → Prop)]
-    [Std.Antisymm (¬ · < · : β → β → Prop)]
-    {l₁ l₂ : List α} {f : α → β} (w : ∀ x y, x < y → f x < f y) (h : l₁ ≤ l₂) :
-    map f l₁ ≤ map f l₂ := by
-  rw [le_iff_exists] at h ⊢
-  obtain (h | ⟨i, h₁, h₂, w₁, w₂⟩) := h
-  · left
-    rw [h]
-    simp
-  · right
-    refine ⟨i, by simpa using h₁, by simpa using h₂, ?_, ?_⟩
-    · simp +contextual [w₁]
-    · simpa using w _ _ w₂
-
 end List

src/Init/Data/Nat/Div/Lemmas.lean

@@ -26,11 +26,11 @@ theorem div_le_iff_le_mul (h : 0 < k) : x / k ≤ y ↔ x ≤ y * k + k - 1 := b
   omega
 
 -- TODO: reprove `div_eq_of_lt_le` in terms of this:
-protected theorem div_eq_iff (h : 0 < k) : x / k = y ↔ x ≤ y * k + k - 1 ∧ y * k ≤ x := by
+theorem div_eq_iff (h : 0 < k) : x / k = y ↔ x ≤ y * k + k - 1 ∧ y * k ≤ x := by
   rw [Nat.eq_iff_le_and_ge, le_div_iff_mul_le h, Nat.div_le_iff_le_mul h]
 
 theorem lt_of_div_eq_zero (h : 0 < k) (h' : x / k = 0) : x < k := by
-  rw [Nat.div_eq_iff h] at h'
+  rw [div_eq_iff h] at h'
   omega
 
 theorem div_eq_zero_iff_lt (h : 0 < k) : x / k = 0 ↔ x < k :=

src/Init/Data/String/Lemmas.lean

@@ -23,7 +23,7 @@ attribute [local instance] Char.notLTTrans Char.notLTAntisymm Char.notLTTotal
 
 protected theorem le_trans {a b c : String} : a ≤ b → b ≤ c → a ≤ c := List.le_trans
 protected theorem lt_trans {a b c : String} : a < b → b < c → a < c := List.lt_trans
-protected theorem le_total (a b : String) : a ≤ b ∨ b ≤ a := List.le_total _ _
+protected theorem le_total (a b : String) : a ≤ b ∨ b ≤ a := List.le_total
 protected theorem le_antisymm {a b : String} : a ≤ b → b ≤ a → a = b := fun h₁ h₂ => String.ext (List.le_antisymm (as := a.data) (bs := b.data) h₁ h₂)
 protected theorem lt_asymm {a b : String} (h : a < b) : ¬ b < a := List.lt_asymm h
 protected theorem ne_of_lt {a b : String} (h : a < b) : a ≠ b := by

src/Init/Data/Vector/Lemmas.lean

@@ -247,7 +247,7 @@ theorem toArray_mk (a : Array α) (h : a.size = n) : (Vector.mk a h).toArray = a
 
 @[simp] theorem toArray_mkVector : (mkVector n a).toArray = mkArray n a := rfl
 
-@[simp] theorem toArray_inj {v w : Vector α n} : v.toArray = w.toArray ↔ v = w := by
+theorem toArray_inj {v w : Vector α n} : v.toArray = w.toArray ↔ v = w := by
   cases v
   cases w
   simp

src/Init/Data/Vector/Lex.lean

@@ -57,7 +57,6 @@ protected theorem lt_irrefl [LT α] [Std.Irrefl (· < · : α → α → Prop)]
 instance ltIrrefl [LT α] [Std.Irrefl (· < · : α → α → Prop)] : Std.Irrefl (α := Vector α n) (· < ·) where
   irrefl := Vector.lt_irrefl
 
-@[simp] theorem not_lt_empty [LT α] (l : Vector α 0) : ¬ l < #v[] := Array.not_lt_empty l.toArray
 @[simp] theorem empty_le [LT α] (l : Vector α 0) : #v[] ≤ l := Array.empty_le l.toArray
 
 @[simp] theorem le_empty [LT α] (l : Vector α 0) : l ≤ #v[] ↔ l = #v[] := by
@@ -70,12 +69,12 @@ protected theorem le_refl [LT α] [i₀ : Std.Irrefl (· < · : α → α → Pr
 instance [LT α] [Std.Irrefl (· < · : α → α → Prop)] : Std.Refl (· ≤ · : Vector α n → Vector α n → Prop) where
   refl := Vector.le_refl
 
-protected theorem lt_trans [LT α]
+protected theorem lt_trans [LT α] [DecidableLT α]
     [i₁ : Trans (· < · : α → α → Prop) (· < ·) (· < ·)]
     {l₁ l₂ l₃ : Vector α n} (h₁ : l₁ < l₂) (h₂ : l₂ < l₃) : l₁ < l₃ :=
   Array.lt_trans h₁ h₂
 
-instance [LT α]
+instance [LT α] [DecidableLT α]
     [Trans (· < · : α → α → Prop) (· < ·) (· < ·)] :
     Trans (· < · : Vector α n → Vector α n → Prop) (· < ·) (· < ·) where
   trans h₁ h₂ := Vector.lt_trans h₁ h₂
@@ -104,7 +103,7 @@ instance [DecidableEq α] [LT α] [DecidableLT α]
     Trans (· ≤ · : Vector α n → Vector α n → Prop) (· ≤ ·) (· ≤ ·) where
   trans h₁ h₂ := Vector.le_trans h₁ h₂
 
-protected theorem lt_asymm [LT α]
+protected theorem lt_asymm [DecidableEq α] [LT α] [DecidableLT α]
     [i : Std.Asymm (· < · : α → α → Prop)]
     {l₁ l₂ : Vector α n} (h : l₁ < l₂) : ¬ l₂ < l₁ := Array.lt_asymm h
 
@@ -114,31 +113,13 @@ instance [DecidableEq α] [LT α] [DecidableLT α]
   asymm _ _ := Vector.lt_asymm
 
 protected theorem le_total [DecidableEq α] [LT α] [DecidableLT α]
-    [i : Std.Total (¬ · < · : α → α → Prop)] (l₁ l₂ : Vector α n) : l₁ ≤ l₂ ∨ l₂ ≤ l₁ :=
-  Array.le_total _ _
+    [i : Std.Total (¬ · < · : α → α → Prop)] {l₁ l₂ : Vector α n} : l₁ ≤ l₂ ∨ l₂ ≤ l₁ :=
+  Array.le_total
 
 instance [DecidableEq α] [LT α] [DecidableLT α]
     [Std.Total (¬ · < · : α → α → Prop)] :
     Std.Total (· ≤ · : Vector α n → Vector α n → Prop) where
-  total := Vector.le_total
-
-@[simp] protected theorem not_lt [LT α]
-    {l₁ l₂ : Vector α n} : ¬ l₁ < l₂ ↔ l₂ ≤ l₁ := Iff.rfl
-
-@[simp] protected theorem not_le [DecidableEq α] [LT α] [DecidableLT α]
-    {l₁ l₂ : Vector α n} : ¬ l₂ ≤ l₁ ↔ l₁ < l₂ := Decidable.not_not
-
-protected theorem le_of_lt [DecidableEq α] [LT α] [DecidableLT α]
-    [i : Std.Total (¬ · < · : α → α → Prop)]
-    {l₁ l₂ : Vector α n} (h : l₁ < l₂) : l₁ ≤ l₂ :=
-  Array.le_of_lt h
-
-theorem le_iff_lt_or_eq [DecidableEq α] [LT α] [DecidableLT α]
-    [Std.Irrefl (· < · : α → α → Prop)]
-    [Std.Antisymm (¬ · < · : α → α → Prop)]
-    [Std.Total (¬ · < · : α → α → Prop)]
-    {l₁ l₂ : Vector α n} : l₁ ≤ l₂ ↔ l₁ < l₂ ∨ l₁ = l₂ := by
-  simpa using Array.le_iff_lt_or_eq (l₁ := l₁.toArray) (l₂ := l₂.toArray)
+  total _ _ := Vector.le_total
 
 @[simp] theorem lex_eq_true_iff_lt [DecidableEq α] [LT α] [DecidableLT α]
     {l₁ l₂ : Vector α n} : lex l₁ l₂ = true ↔ l₁ < l₂ := by
@@ -218,32 +199,4 @@ theorem le_iff_exists [DecidableEq α] [LT α] [DecidableLT α]
   rcases l₂ with ⟨l₂, n₂⟩
   simp [Array.le_iff_exists, ← n₂]
 
-theorem append_left_lt [LT α] {l₁ : Vector α n} {l₂ l₃ : Vector α m} (h : l₂ < l₃) :
-    l₁ ++ l₂ < l₁ ++ l₃ := by
-  simpa using Array.append_left_lt h
-
-theorem append_left_le [DecidableEq α] [LT α] [DecidableLT α]
-    [Std.Irrefl (· < · : α → α → Prop)]
-    [Std.Asymm (· < · : α → α → Prop)]
-    [Std.Antisymm (¬ · < · : α → α → Prop)]
-    {l₁ : Vector α n} {l₂ l₃ : Vector α m} (h : l₂ ≤ l₃) :
-    l₁ ++ l₂ ≤ l₁ ++ l₃ := by
-  simpa using Array.append_left_le h
-
-theorem map_lt [LT α] [LT β]
-    {l₁ l₂ : Vector α n} {f : α → β} (w : ∀ x y, x < y → f x < f y) (h : l₁ < l₂) :
-    map f l₁ < map f l₂ := by
-  simpa using Array.map_lt w h
-
-theorem map_le [DecidableEq α] [LT α] [DecidableLT α] [DecidableEq β] [LT β] [DecidableLT β]
-    [Std.Irrefl (· < · : α → α → Prop)]
-    [Std.Asymm (· < · : α → α → Prop)]
-    [Std.Antisymm (¬ · < · : α → α → Prop)]
-    [Std.Irrefl (· < · : β → β → Prop)]
-    [Std.Asymm (· < · : β → β → Prop)]
-    [Std.Antisymm (¬ · < · : β → β → Prop)]
-    {l₁ l₂ : Vector α n} {f : α → β} (w : ∀ x y, x < y → f x < f y) (h : l₁ ≤ l₂) :
-    map f l₁ ≤ map f l₂ := by
-  simpa using Array.map_le w h
-
 end Vector

src/Lean/Compiler/LCNF/Probing.lean

@@ -170,14 +170,6 @@ def tail (n : Nat) : Probe α α := fun data => return data[data.size - n:]
 @[inline]
 def head (n : Nat) : Probe α α := fun data => return data[:n]
 
-def runOnDeclsNamed (declNames : Array Name) (probe : Probe Decl β) (phase : Phase := Phase.base): CoreM (Array β) := do
-  let ext := getExt phase
-  let env ← getEnv
-  let decls ← declNames.mapM fun name => do
-    let some decl := getDeclCore? env ext name | throwError "decl `{name}` not found"
-    return decl
-  probe decls |>.run (phase := phase)
-
 def runOnModule (moduleName : Name) (probe : Probe Decl β) (phase : Phase := Phase.base): CoreM (Array β) := do
   let ext := getExt phase
   let env ← getEnv

src/Lean/Compiler/LCNF/ToMono.lean

@@ -50,10 +50,6 @@ partial def LetValue.toMono (e : LetValue) : ToMonoM LetValue := do
       return .const ``Bool.true [] #[]
     else if declName == ``Decidable.isFalse then
       return .const ``Bool.false [] #[]
-    else if declName == ``Decidable.decide then
-      -- Decidable.decide is the identity function since Decidable
-      -- and Bool have the same runtime representation.
-      return args[1]!.toLetValue
     else if let some e' ← isTrivialConstructorApp? declName args then
       e'.toMono
     else if let some (.ctorInfo ctorInfo) := (← getEnv).find? declName then

src/Lean/Elab/App.lean

@@ -628,7 +628,7 @@ mutual
           (in particular, `Lean.Elab.Term.withReuseContext`) controls the ref to avoid leakage of outside data.
           Note that `tacticSyntax` contains no position information itself, since it is erased by `Lean.Elab.Term.quoteAutoTactic`.
           -/
-          let info := (← getRef).getHeadInfo.nonCanonicalSynthetic
+          let info := (← getRef).getHeadInfo
           let tacticBlock := tacticBlock.raw.rewriteBottomUp (·.setInfo info)
           let mvar ← mkTacticMVar argType.consumeTypeAnnotations tacticBlock (.autoParam argName)
           -- Note(kmill): We are adding terminfo to simulate a previous implementation that elaborated `tacticBlock`.

src/Lean/Elab/Print.lean

@@ -121,7 +121,7 @@ private def printStructure (id : Name) (levelParams : List Name) (numParams : Na
       -- Resolution order
       let resOrder ← getStructureResolutionOrder id
       if resOrder.size > 1 then
-        m := m ++ Format.line ++ "field notation resolution order:"
+        m := m ++ Format.line ++ "resolution order:"
           ++ indentD (MessageData.joinSep (resOrder.map (.ofConstName · (fullNames := true))).toList ", ")
       logInfo m
 

src/Lean/Elab/StructInst.lean

@@ -853,7 +853,7 @@ private partial def elabStructInstView (s : StructInstView) (expectedType? : Opt
               let stx ← `(by $tacticSyntax)
               -- See comment in `Lean.Elab.Term.ElabAppArgs.processExplicitArg` about `tacticSyntax`.
               -- We add info to get reliable positions for messages from evaluating the tactic script.
-              let info := field.ref.getHeadInfo.nonCanonicalSynthetic
+              let info := field.ref.getHeadInfo
               let stx := stx.raw.rewriteBottomUp (·.setInfo info)
               let type := (d.getArg! 0).consumeTypeAnnotations
               let mvar ← mkTacticMVar type stx (.fieldAutoParam fieldName s.structName)

src/Lean/Meta/Closure.lean

@@ -203,34 +203,9 @@ partial def collectExprAux (e : Expr) : ClosureM Expr := do
     let type ← collect type
     let newFVarId ← mkFreshFVarId
     let userName ← mkNextUserName
-    /-
-    Recall that delayed assignment metavariables must always be applied to at least
-    `a.fvars.size` arguments (where `a : DelayedMetavarAssignment` is its record).
-    This assumption is used in `lean::instantiate_mvars_fn::visit_app` for example, where there's a comment
-    about how under-applied delayed assignments are an error.
-
-    If we were to collect the delayed assignment metavariable itself and push it onto the `exprMVarArgs` list,
-    then `exprArgs` returned by `Lean.Meta.Closure.mkValueTypeClosure` would contain underapplied delayed assignment metavariables.
-    This leads to kernel 'declaration has metavariables' errors, as reported in https://github.com/leanprover/lean4/issues/6354
-
-    The straightforward solution to this problem (implemented below) is to eta expand the delayed assignment metavariable
-    to ensure it is fully applied. This isn't full eta expansion; we only need to eta expand the first `fvars.size` arguments.
-
-    Note: there is the possibility of handling special cases to create more-efficient terms.
-    For example, if the delayed assignment metavariable is applied to fvars, we could avoid eta expansion for those arguments
-    since the fvars are being collected anyway. It's not clear that the additional implementation complexity is worth it,
-    and it is something we can evaluate later. In any case, the current solution is necessary as the generic case.
-    -/
-    let e' ←
-      if let some { fvars, .. } ← getDelayedMVarAssignment? mvarId then
-        -- Eta expand `e` for the requisite number of arguments.
-        forallBoundedTelescope mvarDecl.type fvars.size fun args _ => do
-          mkLambdaFVars args <| mkAppN e args
-      else
-        pure e
     modify fun s => { s with
       newLocalDeclsForMVars := s.newLocalDeclsForMVars.push $ .cdecl default newFVarId userName type .default .default,
-      exprMVarArgs          := s.exprMVarArgs.push e'
+      exprMVarArgs          := s.exprMVarArgs.push e
     }
     return mkFVar newFVarId
   | Expr.fvar fvarId =>

src/Lean/Meta/CongrTheorems.lean

@@ -4,9 +4,6 @@ Released under Apache 2.0 license as described in the file LICENSE.
 Authors: Leonardo de Moura
 -/
 prelude
-import Lean.AddDecl
-import Lean.ReservedNameAction
-import Lean.ResolveName
 import Lean.Meta.AppBuilder
 import Lean.Class
 
@@ -35,7 +32,7 @@ inductive CongrArgKind where
   For congr-simp theorems only.  Indicates a decidable instance argument.
   The lemma contains two arguments [a_i : Decidable ...] [b_i : Decidable ...] -/
   | subsingletonInst
-  deriving Inhabited, Repr, BEq
+  deriving Inhabited, Repr
 
 structure CongrTheorem where
   type     : Expr
@@ -355,89 +352,4 @@ def mkCongrSimp? (f : Expr) (subsingletonInstImplicitRhs : Bool := true) : MetaM
   let info ← getFunInfo f
   mkCongrSimpCore? f info (← getCongrSimpKinds f info) (subsingletonInstImplicitRhs := subsingletonInstImplicitRhs)
 
-def hcongrThmSuffixBase := "hcongr"
-def hcongrThmSuffixBasePrefix := hcongrThmSuffixBase ++ "_"
-
-/-- Returns `true` if `s` is of the form `hcongr_<idx>` -/
-def isHCongrReservedNameSuffix (s : String) : Bool :=
-  hcongrThmSuffixBasePrefix.isPrefixOf s && (s.drop 7).isNat
-
-def congrSimpSuffix := "congr_simp"
-
-builtin_initialize congrKindsExt : MapDeclarationExtension (Array CongrArgKind) ← mkMapDeclarationExtension
-
-builtin_initialize registerReservedNamePredicate fun env n =>
-  match n with
-  | .str p s => (isHCongrReservedNameSuffix s || s == congrSimpSuffix) && env.isSafeDefinition p
-  | _ => false
-
-builtin_initialize
-  registerReservedNameAction fun name => do
-    let .str p s := name | return false
-    unless (← getEnv).isSafeDefinition p do return false
-    if isHCongrReservedNameSuffix s then
-      let numArgs := (s.drop 7).toNat!
-      try MetaM.run' do
-        let info ← getConstInfo p
-        let f := mkConst p (info.levelParams.map mkLevelParam)
-        let congrThm ← mkHCongrWithArity f numArgs
-        addDecl <| Declaration.thmDecl {
-           name, type := congrThm.type, value := congrThm.proof
-           levelParams := info.levelParams
-        }
-        modifyEnv fun env => congrKindsExt.insert env name congrThm.argKinds
-        return true
-      catch _ => return false
-    else if s == congrSimpSuffix then
-      try MetaM.run' do
-        let cinfo ← getConstInfo p
-        let f := mkConst p (cinfo.levelParams.map mkLevelParam)
-        let info ← getFunInfo f
-        let some congrThm ← mkCongrSimpCore? f info (← getCongrSimpKinds f info)
-          | return false
-        addDecl <| Declaration.thmDecl {
-           name, type := congrThm.type, value := congrThm.proof
-           levelParams := cinfo.levelParams
-        }
-        modifyEnv fun env => congrKindsExt.insert env name congrThm.argKinds
-        return true
-      catch _ => return false
-    else
-      return false
-
-/--
-Similar to `mkHCongrWithArity`, but uses reserved names to ensure we don't keep creating the
-same congruence theorem over and over again.
--/
-def mkHCongrWithArityForConst? (declName : Name) (levels : List Level) (numArgs : Nat) : MetaM (Option CongrTheorem) := do
-  try
-    let suffix := hcongrThmSuffixBasePrefix ++ toString numArgs
-    let thmName := Name.str declName suffix
-    unless (← getEnv).contains thmName do
-      executeReservedNameAction thmName
-    let proof := mkConst thmName levels
-    let type ← inferType proof
-    let some argKinds := congrKindsExt.getState (← getEnv) |>.find? thmName
-      | unreachable!
-    return some { proof, type, argKinds }
-  catch _ =>
-    return none
-
-/--
-Similar to `mkCongrSimp?`, but uses reserved names to ensure we don't keep creating the
-same congruence theorem over and over again.
--/
-def mkCongrSimpForConst? (declName : Name) (levels : List Level) : MetaM (Option CongrTheorem) := do
-  try
-    let thmName := Name.str declName congrSimpSuffix
-    unless (← getEnv).contains thmName do
-      executeReservedNameAction thmName
-    let proof := mkConst thmName levels
-    let type ← inferType proof
-    let some argKinds := congrKindsExt.getState (← getEnv) |>.find? thmName
-      | unreachable!
-    return some { proof, type, argKinds }
-  catch _ =>
-    return none
-
 end Lean.Meta

src/Lean/Meta/Tactic/Grind.lean

@@ -12,14 +12,3 @@ import Lean.Meta.Tactic.Grind.Util
 import Lean.Meta.Tactic.Grind.Cases
 import Lean.Meta.Tactic.Grind.Injection
 import Lean.Meta.Tactic.Grind.Core
-
-namespace Lean
-
-builtin_initialize registerTraceClass `grind
-builtin_initialize registerTraceClass `grind.eq
-builtin_initialize registerTraceClass `grind.issues
-builtin_initialize registerTraceClass `grind.add
-builtin_initialize registerTraceClass `grind.pre
-builtin_initialize registerTraceClass `grind.debug
-
-end Lean

src/Lean/Meta/Tactic/Grind/Core.lean

@@ -8,71 +8,6 @@ import Lean.Meta.Tactic.Grind.Types
 import Lean.Meta.LitValues
 
 namespace Lean.Meta.Grind
-/-- Helper function for pretty printing the state for debugging purposes. -/
-def ppENodeRef (e : Expr) : GoalM Format := do
-  let some n ← getENode? e | return "_"
-  return f!"#{n.idx}"
-
-/-- Returns expressions in the given expression equivalence class. -/
-partial def getEqc (e : Expr) : GoalM (List Expr) :=
-  go e e []
-where
-  go (first : Expr) (e : Expr) (acc : List Expr) : GoalM (List Expr) := do
-    let next ← getNext e
-    let acc := e :: acc
-    if isSameExpr first next then
-      return acc
-    else
-      go first next acc
-
-/-- Returns all equivalence classes in the current goal. -/
-partial def getEqcs : GoalM (List (List Expr)) := do
-  let mut r := []
-  for (_, node) in (← get).enodes do
-    if isSameExpr node.root node.self then
-      r := (← getEqc node.self) :: r
-  return r
-
-/-- Helper function for pretty printing the state for debugging purposes. -/
-def ppENodeDeclValue (e : Expr) : GoalM Format := do
-  if e.isApp && !(← isLitValue e) then
-    e.withApp fun f args => do
-      let r ← if f.isConst then
-        ppExpr f
-      else
-        ppENodeRef f
-      let mut r := r
-      for arg in args do
-        r := r ++ " " ++ (← ppENodeRef arg)
-      return r
-  else
-    ppExpr e
-
-/-- Helper function for pretty printing the state for debugging purposes. -/
-def ppENodeDecl (e : Expr) : GoalM Format := do
-  let mut r := f!"{← ppENodeRef e} := {← ppENodeDeclValue e}"
-  let n ← getENode e
-  unless isSameExpr e n.root do
-    r := r ++ f!" ↦ {← ppENodeRef n.root}"
-  if n.interpreted then
-    r := r ++ ", [val]"
-  if n.ctor then
-    r := r ++ ", [ctor]"
-  return r
-
-/-- Pretty print goal state for debugging purposes. -/
-def ppState : GoalM Format := do
-  let mut r := f!"Goal:"
-  let nodes := (← get).enodes.toArray.map (·.2)
-  let nodes := nodes.qsort fun a b => a.idx < b.idx
-  for node in nodes do
-    r := r ++ "\n" ++ (← ppENodeDecl node.self)
-  let eqcs ← getEqcs
-  for eqc in eqcs do
-    if eqc.length > 1 then
-      r := r ++ "\n" ++ "{" ++ (Format.joinSep (← eqc.mapM ppENodeRef) ", ") ++  "}"
-  return r
-
 /--
 Returns `true` if `e` is `True`, `False`, or a literal value.
 See `LitValues` for supported literals.
@@ -85,12 +20,31 @@ def isInterpreted (e : Expr) : MetaM Bool := do
 Creates an `ENode` for `e` if one does not already exist.
 This method assumes `e` has been hashconsed.
 -/
-def mkENode (e : Expr) (generation : Nat) : GoalM Unit := do
-  if (← alreadyInternalized e) then return ()
+def mkENode (e : Expr) (generation : Nat := 0) : GoalM Unit := do
+  if (← getENode? e).isSome then return ()
   let ctor := (← isConstructorAppCore? e).isSome
   let interpreted ← isInterpreted e
   mkENodeCore e interpreted ctor generation
 
+/--
+Returns the root element in the equivalence class of `e`.
+-/
+def getRoot (e : Expr) : GoalM Expr := do
+  let some n ← getENode? e | return e
+  return n.root
+
+/--
+Returns the next element in the equivalence class of `e`.
+-/
+def getNext (e : Expr) : GoalM Expr := do
+  let some n ← getENode? e | return e
+  return n.next
+
+@[inline] def isSameExpr (a b : Expr) : Bool :=
+  -- It is safe to use pointer equality because we hashcons all expressions
+  -- inserted into the E-graph
+  unsafe ptrEq a b
+
 private def pushNewEqCore (lhs rhs proof : Expr) (isHEq : Bool) : GoalM Unit :=
   modify fun s => { s with newEqs := s.newEqs.push { lhs, rhs, proof, isHEq } }
 
@@ -100,49 +54,6 @@ private def pushNewEqCore (lhs rhs proof : Expr) (isHEq : Bool) : GoalM Unit :=
 @[inline] private def pushNewHEq (lhs rhs proof : Expr) : GoalM Unit :=
   pushNewEqCore lhs rhs proof (isHEq := true)
 
-/--
-Adds `e` to congruence table.
--/
-def addCongrTable (_e : Expr) : GoalM Unit := do
-  -- TODO
-  return ()
-
-partial def internalize (e : Expr) (generation : Nat) : GoalM Unit := do
-  if (← alreadyInternalized e) then return ()
-  match e with
-  | .bvar .. => unreachable!
-  | .sort .. => return ()
-  | .fvar .. | .letE .. | .lam .. | .forallE .. =>
-    mkENodeCore e (ctor := false) (interpreted := false) (generation := generation)
-  | .lit .. | .const .. =>
-    mkENode e generation
-  | .mvar ..
-  | .mdata ..
-  | .proj .. =>
-    trace[grind.issues] "unexpected term during internalization{indentExpr e}"
-    mkENodeCore e (ctor := false) (interpreted := false) (generation := generation)
-  | .app .. =>
-    if (← isLitValue e) then
-      -- We do not want to internalize the components of a literal value.
-      mkENode e generation
-    else e.withApp fun f args => do
-      unless f.isConst do
-        internalize f generation
-      let info ← getFunInfo f
-      let shouldInternalize (i : Nat) : GoalM Bool := do
-        if h : i < info.paramInfo.size then
-          let pinfo := info.paramInfo[i]
-          if pinfo.binderInfo.isInstImplicit || pinfo.isProp then
-            return false
-        return true
-      for h : i in [: args.size] do
-        let arg := args[i]
-        if (← shouldInternalize i) then
-          unless (← isTypeFormer arg) do
-            internalize arg generation
-      mkENode e generation
-      addCongrTable e
-
 /--
 The fields `target?` and `proof?` in `e`'s `ENode` are encoding a transitivity proof
 from `e` to the root of the equivalence class
@@ -154,7 +65,7 @@ private partial def invertTrans (e : Expr) : GoalM Unit := do
   go e false none none
 where
   go (e : Expr) (flippedNew : Bool) (targetNew? : Option Expr) (proofNew? : Option Expr) : GoalM Unit := do
-    let node ← getENode e
+    let some node ← getENode? e | unreachable!
     if let some target := node.target? then
       go target (!node.flipped) (some e) node.proof?
     setENode e { node with
@@ -166,29 +77,20 @@ where
 private def markAsInconsistent : GoalM Unit :=
   modify fun s => { s with inconsistent := true }
 
-def isInconsistent : GoalM Bool :=
-  return (← get).inconsistent
-
 private partial def addEqStep (lhs rhs proof : Expr) (isHEq : Bool) : GoalM Unit := do
-  trace[grind.eq] "{lhs} {if isHEq then "≡" else "="} {rhs}"
-  let lhsNode ← getENode lhs
-  let rhsNode ← getENode rhs
-  if isSameExpr lhsNode.root rhsNode.root then
-    -- `lhs` and `rhs` are already in the same equivalence class.
-    trace[grind.debug] "{← ppENodeRef lhs} and {← ppENodeRef rhs} are already in the same equivalence class"
-    return ()
-  let lhsRoot ← getENode lhsNode.root
-  let rhsRoot ← getENode rhsNode.root
+  let some lhsNode ← getENode? lhs | return () -- `lhs` has not been internalized yet
+  let some rhsNode ← getENode? rhs | return () -- `rhs` has not been internalized yet
+  if isSameExpr lhsNode.root rhsNode.root then return () -- `lhs` and `rhs` are already in the same equivalence class.
+  let some lhsRoot ← getENode? lhsNode.root | unreachable!
+  let some rhsRoot ← getENode? rhsNode.root | unreachable!
   if    (lhsRoot.interpreted && !rhsRoot.interpreted)
      || (lhsRoot.ctor && !rhsRoot.ctor)
      || (lhsRoot.size > rhsRoot.size && !rhsRoot.interpreted && !rhsRoot.ctor) then
     go rhs lhs rhsNode lhsNode rhsRoot lhsRoot true
   else
     go lhs rhs lhsNode rhsNode lhsRoot rhsRoot false
-  trace[grind.debug] "after addEqStep, {← ppState}"
 where
   go (lhs rhs : Expr) (lhsNode rhsNode lhsRoot rhsRoot : ENode) (flipped : Bool) : GoalM Unit := do
-    trace[grind.debug] "adding {← ppENodeRef lhs} ↦ {← ppENodeRef rhs}"
     let mut valueInconsistency := false
     if lhsRoot.interpreted && rhsRoot.interpreted then
       if lhsNode.root.isTrue || rhsNode.root.isTrue then
@@ -213,11 +115,9 @@ where
     -- TODO: Remove parents from congruence table
     -- TODO: set propagateBool
     updateRoots lhs rhsNode.root true -- TODO
-    trace[grind.debug] "{← ppENodeRef lhs} new root {← ppENodeRef rhsNode.root}, {← ppENodeRef (← getRoot lhs)}"
     -- TODO: Reinsert parents into congruence table
     setENode lhsNode.root { lhsRoot with
       next := rhsRoot.next
-      root := rhsNode.root
     }
     setENode rhsNode.root { rhsRoot with
       next := lhsRoot.next
@@ -230,23 +130,19 @@ where
   updateRoots (lhs : Expr) (rootNew : Expr) (_propagateBool : Bool) : GoalM Unit := do
     let rec loop (e : Expr) : GoalM Unit := do
       -- TODO: propagateBool
-      let n ← getENode e
+      let some n ← getENode? e | unreachable!
       setENode e { n with root := rootNew }
       if isSameExpr lhs n.next then return ()
       loop n.next
     loop lhs
 
-/-- Ensures collection of equations to be processed is empty. -/
-def resetNewEqs : GoalM Unit :=
-  modify fun s => { s with newEqs := #[] }
-
 partial def addEqCore (lhs rhs proof : Expr) (isHEq : Bool) : GoalM Unit := do
   addEqStep lhs rhs proof isHEq
   processTodo
 where
   processTodo : GoalM Unit := do
-    if (← isInconsistent) then
-      resetNewEqs
+    if (← get).inconsistent then
+      modify fun s => { s with newEqs := #[] }
       return ()
     let some { lhs, rhs, proof, isHEq } := (← get).newEqs.back? | return ()
     addEqStep lhs rhs proof isHEq
@@ -258,42 +154,4 @@ def addEq (lhs rhs proof : Expr) : GoalM Unit := do
 def addHEq (lhs rhs proof : Expr) : GoalM Unit := do
   addEqCore lhs rhs proof true
 
-/--
-Adds a new `fact` justified by the given proof and using the given generation.
--/
-def add (fact : Expr) (proof : Expr) (generation := 0) : GoalM Unit := do
-  trace[grind.add] "{proof} : {fact}"
-  if (← isInconsistent) then return ()
-  resetNewEqs
-  let_expr Not p := fact
-    | go fact false
-  go p true
-where
-  go (p : Expr) (isNeg : Bool) : GoalM Unit := do
-    trace[grind.add] "isNeg: {isNeg}, {p}"
-    match_expr p with
-    | Eq _ lhs rhs => goEq p lhs rhs isNeg false
-    | HEq _ _ lhs rhs => goEq p lhs rhs isNeg true
-    | _ =>
-      internalize p generation
-      if isNeg then
-        addEq p (← getFalseExpr) (← mkEqFalse proof)
-      else
-        addEq p (← getTrueExpr) (← mkEqTrue proof)
-
-  goEq (p : Expr) (lhs rhs : Expr) (isNeg : Bool) (isHEq : Bool) : GoalM Unit := do
-    if isNeg then
-      internalize p generation
-      addEq p (← getFalseExpr) (← mkEqFalse proof)
-    else
-      internalize lhs generation
-      internalize rhs generation
-      addEqCore lhs rhs proof isHEq
-
-/--
-Adds a new hypothesis.
--/
-def addHyp (fvarId : FVarId) (generation := 0) : GoalM Unit := do
-  add (← fvarId.getType) (mkFVar fvarId) generation
-
 end Lean.Meta.Grind

src/Lean/Meta/Tactic/Grind/Preprocessor.lean

@@ -68,11 +68,8 @@ def introNext (goal : Goal) : PreM IntroResult := do
       else
         let tag ← goal.mvarId.getTag
         let q := target.bindingBody!
-        -- TODO: keep applying simp/eraseIrrelevantMData/canon/shareCommon until no progress
         let r ← simp goal p
         let p' := r.expr
-        let p' ← eraseIrrelevantMData p'
-        let p' ← foldProjs p'
         let p' ← canon p'
         let p' ← shareCommon p'
         let fvarId ← mkFreshFVarId
@@ -136,7 +133,8 @@ partial def loop (goal : Goal) : PreM Unit := do
     else if let some goal ← applyInjection? goal fvarId then
       loop goal
     else
-      loop (← GoalM.run' goal <| addHyp fvarId)
+      let clause ← goal.mvarId.withContext do mkInputClause fvarId
+      loop { goal with clauses := goal.clauses.push clause }
   | .newDepHyp goal =>
     loop goal
   | .newLocal fvarId goal =>
@@ -145,17 +143,8 @@ partial def loop (goal : Goal) : PreM Unit := do
     else
       loop goal
 
-def ppGoals : PreM Format := do
-  let mut r := f!""
-  for goal in (← get).goals do
-    let (f, _) ← GoalM.run goal ppState
-    r := r ++ Format.line ++ f
-  return r
-
 def preprocess (mvarId : MVarId) : PreM State := do
   loop (← mkGoal mvarId)
-  if (← isTracingEnabledFor `grind.pre) then
-    trace[grind.pre] (← ppGoals)
   get
 
 end Preprocessor
@@ -164,7 +153,6 @@ open Preprocessor
 
 partial def main (mvarId : MVarId) (mainDeclName : Name) : MetaM (List MVarId) := do
   mvarId.ensureProp
-  -- TODO: abstract metavars
   mvarId.ensureNoMVar
   let mvarId ← mvarId.clearAuxDecls
   let mvarId ← mvarId.revertAll

src/Lean/Meta/Tactic/Grind/Types.lean

@@ -6,90 +6,38 @@ Authors: Leonardo de Moura
 prelude
 import Lean.Util.ShareCommon
 import Lean.Meta.Basic
-import Lean.Meta.CongrTheorems
 import Lean.Meta.AbstractNestedProofs
 import Lean.Meta.Canonicalizer
 import Lean.Meta.Tactic.Util
 
 namespace Lean.Meta.Grind
 
-@[inline] def isSameExpr (a b : Expr) : Bool :=
-  -- It is safe to use pointer equality because we hashcons all expressions
-  -- inserted into the E-graph
-  unsafe ptrEq a b
-
 structure Context where
   mainDeclName : Name
 
-/--
-Key for the congruence theorem cache.
--/
-structure CongrTheoremCacheKey where
-  f       : Expr
-  numArgs : Nat
-
--- We manually define `BEq` because we wannt to use pointer equality.
-instance : BEq CongrTheoremCacheKey where
-  beq a b := isSameExpr a.f b.f && a.numArgs == b.numArgs
-
--- We manually define `Hashable` because we wannt to use pointer equality.
-instance : Hashable CongrTheoremCacheKey where
-  hash a := mixHash (unsafe ptrAddrUnsafe a.f).toUInt64 (hash a.numArgs)
-
 structure State where
   canon      : Canonicalizer.State := {}
   /-- `ShareCommon` (aka `Hashconsing`) state. -/
   scState    : ShareCommon.State.{0} ShareCommon.objectFactory := ShareCommon.State.mk _
   /-- Next index for creating auxiliary theorems. -/
   nextThmIdx : Nat := 1
-  /--
-  Congruence theorems generated so far. Recall that for constant symbols
-  we rely on the reserved name feature (i.e., `mkHCongrWithArityForConst?`).
-  Remark: we currently do not reuse congruence theorems
-  -/
-  congrThms  : PHashMap CongrTheoremCacheKey CongrTheorem := {}
-  trueExpr   : Expr
-  falseExpr  : Expr
 
 abbrev GrindM := ReaderT Context $ StateRefT State MetaM
 
-def GrindM.run (x : GrindM α) (mainDeclName : Name) : MetaM α := do
-  let scState := ShareCommon.State.mk _
-  let (falseExpr, scState) := ShareCommon.State.shareCommon scState (mkConst ``False)
-  let (trueExpr, scState)  := ShareCommon.State.shareCommon scState (mkConst ``True)
-  x { mainDeclName } |>.run' { scState, trueExpr, falseExpr }
+@[inline] def GrindM.run (x : GrindM α) (mainDeclName : Name) : MetaM α :=
+  x { mainDeclName } |>.run' {}
 
-def getTrueExpr : GrindM Expr := do
-  return (← get).trueExpr
-
-def getFalseExpr : GrindM Expr := do
-  return (← get).falseExpr
-
-/--
-Abtracts nested proofs in `e`. This is a preprocessing step performed before internalization.
--/
 def abstractNestedProofs (e : Expr) : GrindM Expr := do
   let nextIdx := (← get).nextThmIdx
   let (e, s') ← AbstractNestedProofs.visit e |>.run { baseName := (← read).mainDeclName } |>.run |>.run { nextIdx }
   modify fun s => { s with nextThmIdx := s'.nextIdx }
   return e
 
-/--
-Applies hash-consing to `e`. Recall that all expressions in a `grind` goal have
-been hash-consing. We perform this step before we internalize expressions.
--/
 def shareCommon (e : Expr) : GrindM Expr := do
-  modifyGet fun { canon, scState, nextThmIdx, congrThms, trueExpr, falseExpr } =>
+  modifyGet fun { canon, scState, nextThmIdx } =>
     let (e, scState) := ShareCommon.State.shareCommon scState e
-    (e, { canon, scState, nextThmIdx, congrThms, trueExpr, falseExpr })
+    (e, { canon, scState, nextThmIdx })
 
-/--
-Applies the canonicalizer to all subterms of `e`.
--/
--- TODO: the current canonicalizer is not a good solution for `grind`.
--- The problem is that two different applications `@f inst_1 a` and `@f inst_2 b`
--- may still have syntaticaally different instances. Thus, if we learn that `a = b`,
--- congruence closure will fail to see that the two applications are congruent.
 def canon (e : Expr) : GrindM Expr := do
   let canonS ← modifyGet fun s => (s.canon, { s with canon := {} })
   let (e, canonS) ← Canonicalizer.CanonM.run (canonRec e) (s := canonS)
@@ -104,28 +52,11 @@ where
         return .done e
     transform e post
 
-/--
-Creates a congruence theorem for a `f`-applications with `numArgs` arguments.
--/
-def mkHCongrWithArity (f : Expr) (numArgs : Nat) : GrindM CongrTheorem := do
-  let key := { f, numArgs }
-  if let some result := (← get).congrThms.find? key then
-    return result
-  if let .const declName us := f then
-    if let some result ← mkHCongrWithArityForConst? declName us numArgs then
-      modify fun s => { s with congrThms := s.congrThms.insert key result }
-      return result
-  let result ← Meta.mkHCongrWithArity f numArgs
-  modify fun s => { s with congrThms := s.congrThms.insert key result }
-  return result
-
 /--
 Stores information for a node in the egraph.
 Each internalized expression `e` has an `ENode` associated with it.
 -/
 structure ENode where
-  /-- Node represented by this ENode. -/
-  self : Expr
   /-- Next element in the equivalence class. -/
   next : Expr
   /-- Root (aka canonical representative) of the equivalence class -/
@@ -153,15 +84,18 @@ structure ENode where
   on heterogeneous equality.
   -/
   heqProofs : Bool := false
-  /--
-  Unique index used for pretty printing and debugging purposes.
-  -/
-  idx : Nat := 0
   generation : Nat := 0
   /-- Modification time -/
   mt : Nat := 0
   -- TODO: see Lean 3 implementation
-  deriving Inhabited, Repr
+
+structure Clause where
+  expr  : Expr
+  proof : Expr
+  deriving Inhabited
+
+def mkInputClause (fvarId : FVarId) : MetaM Clause :=
+  return { expr := (← fvarId.getType), proof := mkFVar fvarId }
 
 structure NewEq where
   lhs   : Expr
@@ -171,15 +105,13 @@ structure NewEq where
 
 structure Goal where
   mvarId       : MVarId
+  clauses      : PArray Clause := {}
   enodes       : PHashMap USize ENode := {}
-  /-- Equations to be processed. -/
   newEqs       : Array NewEq := #[]
   /-- `inconsistent := true` if `ENode`s for `True` and `False` are in the same equivalence class. -/
   inconsistent : Bool := false
   /-- Goal modification time. -/
   gmt          : Nat := 0
-  /-- Next unique index for creating ENodes -/
-  nextIdx      : Nat := 0
   deriving Inhabited
 
 def Goal.admit (goal : Goal) : MetaM Unit :=
@@ -188,10 +120,10 @@ def Goal.admit (goal : Goal) : MetaM Unit :=
 abbrev GoalM := StateRefT Goal GrindM
 
 @[inline] def GoalM.run (goal : Goal) (x : GoalM α) : GrindM (α × Goal) :=
-  goal.mvarId.withContext do StateRefT'.run x goal
+  StateRefT'.run x goal
 
 @[inline] def GoalM.run' (goal : Goal) (x : GoalM Unit) : GrindM Goal :=
-  goal.mvarId.withContext do StateRefT'.run' (x *> get) goal
+  StateRefT'.run' (x *> get) goal
 
 /--
 Returns `some n` if `e` has already been "internalized" into the
@@ -200,43 +132,22 @@ Otherwise, returns `none`s.
 def getENode? (e : Expr) : GoalM (Option ENode) :=
   return (← get).enodes.find? (unsafe ptrAddrUnsafe e)
 
-/-- Returns node associated with `e`. It assumes `e` has already been internalized. -/
-def getENode (e : Expr) : GoalM ENode := do
-  let some n := (← get).enodes.find? (unsafe ptrAddrUnsafe e) | unreachable!
-  return n
-
-/-- Returns the root element in the equivalence class of `e`. -/
-def getRoot (e : Expr) : GoalM Expr :=
-  return (← getENode e).root
-
-/-- Returns the next element in the equivalence class of `e`. -/
-def getNext (e : Expr) : GoalM Expr :=
-  return (← getENode e).next
-
-/-- Returns `true` if `e` has already been internalized. -/
-def alreadyInternalized (e : Expr) : GoalM Bool :=
-  return (← get).enodes.contains (unsafe ptrAddrUnsafe e)
-
 def setENode (e : Expr) (n : ENode) : GoalM Unit :=
   modify fun s => { s with enodes := s.enodes.insert (unsafe ptrAddrUnsafe e) n }
 
 def mkENodeCore (e : Expr) (interpreted ctor : Bool) (generation : Nat) : GoalM Unit := do
   setENode e {
-    self := e, next := e, root := e, cgRoot := e, size := 1
+    next := e, root := e, cgRoot := e, size := 1
     flipped := false
     heqProofs := false
     hasLambdas := e.isLambda
     mt := (← get).gmt
-    idx := (← get).nextIdx
     interpreted, ctor, generation
   }
-  modify fun s => { s with nextIdx := s.nextIdx + 1 }
 
 def mkGoal (mvarId : MVarId) : GrindM Goal := do
-  let trueExpr ← getTrueExpr
-  let falseExpr ← getFalseExpr
   GoalM.run' { mvarId } do
-    mkENodeCore falseExpr (interpreted := true) (ctor := false) (generation := 0)
-    mkENodeCore trueExpr (interpreted := true) (ctor := false) (generation := 0)
+    mkENodeCore (← shareCommon (mkConst ``True)) (interpreted := true) (ctor := false) (generation := 0)
+    mkENodeCore (← shareCommon (mkConst ``False)) (interpreted := true) (ctor := false) (generation := 0)
 
 end Lean.Meta.Grind

src/Lean/Meta/Tactic/Grind/Util.lean

@@ -5,7 +5,6 @@ Authors: Leonardo de Moura
 -/
 prelude
 import Lean.Meta.AbstractNestedProofs
-import Lean.Meta.Transform
 import Lean.Meta.Tactic.Util
 import Lean.Meta.Tactic.Clear
 
@@ -36,7 +35,7 @@ def _root_.Lean.MVarId.transformTarget (mvarId : MVarId) (f : Expr → MetaM Exp
   return mvarNew.mvarId!
 
 /--
-Unfolds all `reducible` declarations occurring in `e`.
+Unfold all `reducible` declarations occurring in `e`.
 -/
 def unfoldReducible (e : Expr) : MetaM Expr :=
   let pre (e : Expr) : MetaM TransformStep := do
@@ -47,25 +46,25 @@ def unfoldReducible (e : Expr) : MetaM Expr :=
   Core.transform e (pre := pre)
 
 /--
-Unfolds all `reducible` declarations occurring in the goal's target.
+Unfold all `reducible` declarations occurring in the goal's target.
 -/
 def _root_.Lean.MVarId.unfoldReducible (mvarId : MVarId) : MetaM MVarId :=
   mvarId.transformTarget Grind.unfoldReducible
 
 /--
-Abstracts nested proofs occurring in the goal's target.
+Abstract nested proofs occurring in the goal's target.
 -/
 def _root_.Lean.MVarId.abstractNestedProofs (mvarId : MVarId) (mainDeclName : Name) : MetaM MVarId :=
   mvarId.transformTarget (Lean.Meta.abstractNestedProofs mainDeclName)
 
 /--
-Beta-reduces the goal's target.
+Beta-reduce the goal's target.
 -/
 def _root_.Lean.MVarId.betaReduce (mvarId : MVarId) : MetaM MVarId :=
   mvarId.transformTarget (Core.betaReduce ·)
 
 /--
-If the target is not `False`, applies `byContradiction`.
+If the target is not `False`, apply `byContradiction`.
 -/
 def _root_.Lean.MVarId.byContra? (mvarId : MVarId) : MetaM (Option MVarId) := mvarId.withContext do
   mvarId.checkNotAssigned `grind.by_contra
@@ -78,7 +77,7 @@ def _root_.Lean.MVarId.byContra? (mvarId : MVarId) : MetaM (Option MVarId) := mv
   return mvarNew.mvarId!
 
 /--
-Clears auxiliary decls used to encode recursive declarations.
+Clear auxiliary decls used to encode recursive declarations.
 `grind` eliminates them to ensure they are not accidentally used by its proof automation.
 -/
 def _root_.Lean.MVarId.clearAuxDecls (mvarId : MVarId) : MetaM MVarId := mvarId.withContext do
@@ -97,31 +96,4 @@ def _root_.Lean.MVarId.clearAuxDecls (mvarId : MVarId) : MetaM MVarId := mvarId.
       throwTacticEx `grind.clear_aux_decls mvarId "failed to clear local auxiliary declaration"
   return mvarId
 
-/--
-In the `grind` tactic, during `Expr` internalization, we don't expect to find `Expr.mdata`.
-This function ensures `Expr.mdata` is not found during internalization.
-Recall that we do not internalize `Expr.forallE` and `Expr.lam` components.
--/
-def eraseIrrelevantMData (e : Expr) : CoreM Expr := do
-  let pre (e : Expr) := do
-    match e with
-    | .letE .. | .lam .. | .forallE .. => return .done e
-    | .mdata _ e => return .continue e
-    | _ => return .continue e
-  Core.transform e (pre := pre)
-
-/--
-Converts nested `Expr.proj`s into projection applications if possible.
--/
-def foldProjs (e : Expr) : MetaM Expr := do
-  let post (e : Expr) := do
-    let .proj structName idx s := e | return .done e
-    let some info := getStructureInfo? (← getEnv) structName | return .done e
-    if h : idx < info.fieldNames.size then
-      let fieldName := info.fieldNames[idx]
-      return .done (← mkProjection s fieldName)
-    else
-      return .done e
-  Meta.transform e (post := post)
-
 end Lean.Meta.Grind

src/Lean/Meta/Transform.lean

@@ -57,13 +57,13 @@ partial def transform {m} [Monad m] [MonadLiftT CoreM m] [MonadControlT CoreM m]
       | .continue e? =>
         let e := e?.getD e
         match e with
-        | .forallE _ d b _ => visitPost (e.updateForallE! (← visit d) (← visit b))
-        | .lam _ d b _     => visitPost (e.updateLambdaE! (← visit d) (← visit b))
-        | .letE _ t v b _  => visitPost (e.updateLet! (← visit t) (← visit v) (← visit b))
-        | .app ..          => e.withApp fun f args => do visitPost (mkAppN (← visit f) (← args.mapM visit))
-        | .mdata _ b       => visitPost (e.updateMData! (← visit b))
-        | .proj _ _ b      => visitPost (e.updateProj! (← visit b))
-        | _                => visitPost e
+        | Expr.forallE _ d b _ => visitPost (e.updateForallE! (← visit d) (← visit b))
+        | Expr.lam _ d b _     => visitPost (e.updateLambdaE! (← visit d) (← visit b))
+        | Expr.letE _ t v b _  => visitPost (e.updateLet! (← visit t) (← visit v) (← visit b))
+        | Expr.app ..          => e.withApp fun f args => do visitPost (mkAppN (← visit f) (← args.mapM visit))
+        | Expr.mdata _ b       => visitPost (e.updateMData! (← visit b))
+        | Expr.proj _ _ b      => visitPost (e.updateProj! (← visit b))
+        | _                    => visitPost e
   visit input |>.run
 
 def betaReduce (e : Expr) : CoreM Expr :=
@@ -99,19 +99,19 @@ partial def transform {m} [Monad m] [MonadLiftT MetaM m] [MonadControlT MetaM m]
         | .continue e? => pure (e?.getD e)
       let rec visitLambda (fvars : Array Expr) (e : Expr) : MonadCacheT ExprStructEq Expr m Expr := do
         match e with
-        | .lam n d b c =>
+        | Expr.lam n d b c =>
           withLocalDecl n c (← visit (d.instantiateRev fvars)) fun x =>
             visitLambda (fvars.push x) b
         | e => visitPost (← mkLambdaFVars (usedLetOnly := usedLetOnly) fvars (← visit (e.instantiateRev fvars)))
       let rec visitForall (fvars : Array Expr) (e : Expr) : MonadCacheT ExprStructEq Expr m Expr := do
         match e with
-        | .forallE n d b c =>
+        | Expr.forallE n d b c =>
           withLocalDecl n c (← visit (d.instantiateRev fvars)) fun x =>
             visitForall (fvars.push x) b
         | e => visitPost (← mkForallFVars (usedLetOnly := usedLetOnly) fvars (← visit (e.instantiateRev fvars)))
       let rec visitLet (fvars : Array Expr) (e : Expr) : MonadCacheT ExprStructEq Expr m Expr := do
         match e with
-        | .letE n t v b _ =>
+        | Expr.letE n t v b _ =>
           withLetDecl n (← visit (t.instantiateRev fvars)) (← visit (v.instantiateRev fvars)) fun x =>
             visitLet (fvars.push x) b
         | e => visitPost (← mkLetFVars (usedLetOnly := usedLetOnly) fvars (← visit (e.instantiateRev fvars)))
@@ -127,22 +127,28 @@ partial def transform {m} [Monad m] [MonadLiftT MetaM m] [MonadControlT MetaM m]
       | .continue e? =>
         let e := e?.getD e
         match e with
-        | .forallE ..    => visitForall #[] e
-        | .lam ..        => visitLambda #[] e
-        | .letE ..       => visitLet #[] e
-        | .app ..        => visitApp e
-        | .mdata _ b     => visitPost (e.updateMData! (← visit b))
-        | .proj _ _ b    => visitPost (e.updateProj! (← visit b))
-        | _              => visitPost e
+        | Expr.forallE ..    => visitForall #[] e
+        | Expr.lam ..        => visitLambda #[] e
+        | Expr.letE ..       => visitLet #[] e
+        | Expr.app ..        => visitApp e
+        | Expr.mdata _ b     => visitPost (e.updateMData! (← visit b))
+        | Expr.proj _ _ b    => visitPost (e.updateProj! (← visit b))
+        | _                  => visitPost e
   visit input |>.run
 
 -- TODO: add options to distinguish zeta and zetaDelta reduction
 def zetaReduce (e : Expr) : MetaM Expr := do
   let pre (e : Expr) : MetaM TransformStep := do
-    let .fvar fvarId := e | return .continue
-    let some localDecl := (← getLCtx).find? fvarId | return .done e
-    let some value := localDecl.value? | return .done e
-    return .visit (← instantiateMVars value)
+    match e with
+    | Expr.fvar fvarId =>
+      match (← getLCtx).find? fvarId with
+      | none => return TransformStep.done e
+      | some localDecl =>
+        if let some value := localDecl.value? then
+          return TransformStep.visit (← instantiateMVars value)
+        else
+          return TransformStep.done e
+    | _ => return .continue
   transform e (pre := pre) (usedLetOnly := true)
 
 /--
@@ -155,9 +161,10 @@ def zetaDeltaFVars (e : Expr) (fvars : Array FVarId) : MetaM Expr :=
     else
       return none
   let pre (e : Expr) : MetaM TransformStep := do
-    let .fvar fvarId := e.getAppFn | return .continue
-    let some val ← unfold? fvarId | return .continue
-    return .visit <| (← instantiateMVars val).beta e.getAppArgs
+    if let .fvar fvarId := e.getAppFn then
+      if let some val ← unfold? fvarId then
+        return .visit <| (← instantiateMVars val).beta e.getAppArgs
+    return .continue
   transform e (pre := pre)
 
 /-- Unfold definitions and theorems in `e` that are not in the current environment, but are in `biggerEnv`. -/
@@ -166,20 +173,25 @@ def unfoldDeclsFrom (biggerEnv : Environment) (e : Expr) : CoreM Expr := do
     let env ← getEnv
     setEnv biggerEnv -- `e` has declarations from `biggerEnv` that are not in `env`
     let pre (e : Expr) : CoreM TransformStep := do
-      let .const declName us := e | return .continue
-      if env.contains declName then
-        return .done e
-      let some info := biggerEnv.find? declName | return .done e
-      if info.hasValue then
-        return .visit (← instantiateValueLevelParams info us)
-      else
-        return .done e
+      match e with
+      | Expr.const declName us .. =>
+        if env.contains declName then
+          return TransformStep.done e
+        else if let some info := biggerEnv.find? declName then
+          if info.hasValue then
+            return TransformStep.visit (← instantiateValueLevelParams info us)
+          else
+            return TransformStep.done e
+        else
+          return TransformStep.done e
+      | _ => return .continue
     Core.transform e (pre := pre)
 
 def eraseInaccessibleAnnotations (e : Expr) : CoreM Expr :=
-  Core.transform e (post := fun e => return .done <| if let some e := inaccessible? e then e else e)
+  Core.transform e (post := fun e => return TransformStep.done <| if let some e := inaccessible? e then e else e)
 
 def erasePatternRefAnnotations (e : Expr) : CoreM Expr :=
-  Core.transform e (post := fun e => return .done <| if let some (_, e) := patternWithRef? e then e else e)
+  Core.transform e (post := fun e => return TransformStep.done <| if let some (_, e) := patternWithRef? e then e else e)
 
-end Lean.Meta
+end Meta
+end Lean

src/Lean/Syntax.lean

@@ -33,16 +33,6 @@ def SourceInfo.updateTrailing (trailing : Substring) : SourceInfo → SourceInfo
 def SourceInfo.getRange? (canonicalOnly := false) (info : SourceInfo) : Option String.Range :=
   return ⟨(← info.getPos? canonicalOnly), (← info.getTailPos? canonicalOnly)⟩
 
-/--
-Converts an `original` or `synthetic (canonical := true)` `SourceInfo` to a
-`synthetic (canonical := false)` `SourceInfo`.
-This is sometimes useful when `SourceInfo` is being moved around between `Syntax`es.
--/
-def SourceInfo.nonCanonicalSynthetic : SourceInfo → SourceInfo
-  | SourceInfo.original _ pos _ endPos => SourceInfo.synthetic pos endPos false
-  | SourceInfo.synthetic pos endPos _  => SourceInfo.synthetic pos endPos false
-  | SourceInfo.none                    => SourceInfo.none
-
 deriving instance BEq for SourceInfo
 
 /-! # Syntax AST -/

src/lake/Lake/Build/Basic.lean

@@ -7,8 +7,6 @@ prelude
 import Lake.Util.Log
 import Lake.Util.Exit
 import Lake.Util.Lift
-import Lake.Util.Task
-import Lake.Util.Opaque
 import Lake.Config.Context
 import Lake.Build.Trace
 
@@ -55,46 +53,10 @@ Whether the build should show progress information.
 def BuildConfig.showProgress (cfg : BuildConfig) : Bool :=
   (cfg.noBuild ∧ cfg.verbosity == .verbose) ∨ cfg.verbosity != .quiet
 
-/-- Information on what this job did. -/
-inductive JobAction
-/-- No information about this job's action is available. -/
-| unknown
-/-- Tried to replay a cached build action (set by `buildFileUnlessUpToDate`) -/
-| replay
-/-- Tried to fetch a build from a store (can be set by `buildUnlessUpToDate?`) -/
-| fetch
-/-- Tried to perform a build action (set by `buildUnlessUpToDate?`) -/
-| build
-deriving Inhabited, Repr, DecidableEq, Ord
-
-instance : LT JobAction := ltOfOrd
-instance : LE JobAction := leOfOrd
-instance : Min JobAction := minOfLe
-instance : Max JobAction := maxOfLe
-
-def JobAction.merge (a b : JobAction) : JobAction :=
-  max a b
-
-/-- Mutable state of a Lake job. -/
-structure JobState where
-  /-- The job's log. -/
-  log : Log := {}
-  /-- Tracks whether this job performed any significant build action. -/
-  action : JobAction := .unknown
-  /-- Current trace of a build job. -/
-  trace : BuildTrace := .nil
-  deriving Inhabited
-
-/-- The result of a Lake job. -/
-abbrev JobResult α := EResult Log.Pos JobState α
-
-/-- The `Task` of a Lake job. -/
-abbrev JobTask α := BaseIOTask (JobResult α)
-
-/-- A Lake job. -/
-structure Job (α : Type u)  where
+/-- The core structure of a Lake job. -/
+structure JobCore (α : Type u)  where
   /-- The Lean `Task` object for the job. -/
-  task : JobTask α
+  task : α
   /--
   A caption for the job in Lake's build monitor.
   Will be formatted like `✔ [3/5] Ran <caption>`.
@@ -104,8 +66,11 @@ structure Job (α : Type u)  where
   optional : Bool := false
   deriving Inhabited
 
-/-- A Lake job with an opaque value in `Type`. -/
-abbrev OpaqueJob := Job Opaque
+/-- A Lake job task with an opaque value type in `Type`. -/
+declare_opaque_type OpaqueJobTask
+
+/-- A Lake job with an opaque value type in `Type`. -/
+abbrev OpaqueJob := JobCore OpaqueJobTask
 
 /-- A Lake context with a build configuration and additional build data. -/
 structure BuildContext extends BuildConfig, Context where

src/lake/Lake/Build/Common.lean

@@ -21,27 +21,11 @@ namespace Lake
 
 /--
 Build trace for the host platform.
-If an artifact includes this trace, it is platform-dependent
+If an artifact includes this in its trace, it is platform-dependent
 and will be rebuilt on different host platforms.
 -/
 def platformTrace := pureHash System.Platform.target
 
-/--
-Mixes the platform into the current job's trace.
-If an artifact includes this trace, it is platform-dependent
-and will be rebuilt on different host platforms.
--/
-@[inline] def addPlatformTrace : JobM PUnit :=
-  addTrace platformTrace
-
-/-- Mixes Lean's trace into the current job's trace. -/
-@[inline] def addLeanTrace : JobM PUnit := do
-  addTrace (← getLeanTrace)
-
-/-- Mixes the trace of a pure value into the current job's trace. -/
-@[inline] def addPureTrace [ComputeHash α Id] (a : α) : JobM PUnit := do
-  addTrace (pureHash a)
-
 /--
 The build trace file format,
 which stores information about a (successful) build.
@@ -203,34 +187,26 @@ def fetchFileTrace (file : FilePath) (text := false) : JobM BuildTrace := do
   return .mk (← fetchFileHash file text) (← getMTime file)
 
 /--
-Builds `file` using `build` unless it already exists and the current job's
-trace matches the trace stored in the `.trace` file. If built, save the new
-trace and cache `file`'s hash in a `.hash` file. Otherwise, try to fetch the
-hash from the `.hash` file using `fetchFileTrace`. Build logs (if any) are
-saved to the trace file and replayed from there if the build is skipped.
+Builds `file` using `build` unless it already exists and `depTrace` matches
+the trace stored in the `.trace` file. If built, save the new `depTrace` and
+cache `file`'s hash in a `.hash` file. Otherwise, try to fetch the hash from
+the `.hash` file using `fetchFileTrace`. Build logs (if any) are saved to
+a `.log.json` file and replayed from there if the build is skipped.
 
-For example, given `file := "foo.c"`, compares `getTrace` with that in
+For example, given `file := "foo.c"`, compares `depTrace` with that in
 `foo.c.trace` with the hash cached in `foo.c.hash` and the log cached in
 `foo.c.trace`.
 
 If `text := true`, `file` is hashed as a text file rather than a binary file.
 -/
-def buildFileUnlessUpToDate'
-  (file : FilePath) (build : JobM PUnit) (text := false)
-: JobM Unit := do
-  let traceFile := FilePath.mk <| file.toString ++ ".trace"
-  buildUnlessUpToDate file (← getTrace) traceFile do
-    build
-    clearFileHash file
-  setTrace (← fetchFileTrace file text)
-
-@[deprecated buildFileUnlessUpToDate' (since := "2024-12-06")]
-abbrev buildFileUnlessUpToDate
+def buildFileUnlessUpToDate
   (file : FilePath) (depTrace : BuildTrace) (build : JobM PUnit) (text := false)
 : JobM BuildTrace := do
-  setTrace depTrace
-  buildFileUnlessUpToDate' file build text
-  getTrace
+  let traceFile := FilePath.mk <| file.toString ++ ".trace"
+  buildUnlessUpToDate file depTrace traceFile do
+    build
+    clearFileHash file
+  fetchFileTrace file text
 
 /--
 Build `file` using `build` after `dep` completes if the dependency's
@@ -239,24 +215,23 @@ trace (and/or `extraDepTrace`) has changed.
 If `text := true`, `file` is handled as a text file rather than a binary file.
 -/
 @[inline] def buildFileAfterDep
-  (file : FilePath) (dep : Job α) (build : α → JobM PUnit)
+  (file : FilePath) (dep : BuildJob α) (build : α → JobM PUnit)
   (extraDepTrace : JobM _ := pure BuildTrace.nil) (text := false)
-: SpawnM (Job FilePath) :=
-  dep.mapM fun depInfo => do
-    addTrace (← extraDepTrace)
-    buildFileUnlessUpToDate' file (build depInfo) text
-    return file
+: SpawnM (BuildJob FilePath) :=
+  dep.bindSync fun depInfo depTrace => do
+    let depTrace := depTrace.mix (← extraDepTrace)
+    let trace ← buildFileUnlessUpToDate file depTrace (build depInfo) text
+    return (file, trace)
 
 /--
 Build `file` using `build` after `deps` have built if any of their traces change.
 
 If `text := true`, `file` is handled as a text file rather than a binary file.
 -/
-@[inline, deprecated buildFileAfterDep (since := "2024-12-06")]
-abbrev buildFileAfterDepList
-  (file : FilePath) (deps : List (Job α)) (build : List α → JobM PUnit)
+@[inline] def buildFileAfterDepList
+  (file : FilePath) (deps : List (BuildJob α)) (build : List α → JobM PUnit)
   (extraDepTrace : JobM _ := pure BuildTrace.nil) (text := false)
-: SpawnM (Job FilePath) := do
+: SpawnM (BuildJob FilePath) := do
   buildFileAfterDep file (.collectList deps) build extraDepTrace text
 
 /--
@@ -264,11 +239,10 @@ Build `file` using `build` after `deps` have built if any of their traces change
 
 If `text := true`, `file` is handled as a text file rather than a binary file.
 -/
-@[inline, deprecated buildFileAfterDep (since := "2024-12-06")]
-def buildFileAfterDepArray
-  (file : FilePath) (deps : Array (Job α)) (build : Array α → JobM PUnit)
+@[inline] def buildFileAfterDepArray
+  (file : FilePath) (deps : Array (BuildJob α)) (build : Array α → JobM PUnit)
   (extraDepTrace : JobM _ := pure BuildTrace.nil) (text := false)
-: SpawnM (Job FilePath) := do
+: SpawnM (BuildJob FilePath) := do
   buildFileAfterDep file (.collectArray deps) build extraDepTrace text
 
 /-! ## Common Builds -/
@@ -278,18 +252,16 @@ A build job for binary file that is expected to already exist (e.g., a data blob
 
 Any byte difference in a binary file will trigger a rebuild of its dependents.
 -/
-def inputBinFile (path : FilePath) : SpawnM (Job FilePath) := Job.async do
-  setTrace (← computeTrace path)
-  return path
+def inputBinFile (path : FilePath) : SpawnM (BuildJob FilePath) :=
+  Job.async <| (path, ·) <$> computeTrace path
 
 /--
 A build job for text file that is expected to already exist (e.g., a source file).
 
 Text file traces have normalized line endings to avoid unnecessary rebuilds across platforms.
 -/
-def inputTextFile (path : FilePath) : SpawnM (Job FilePath) := Job.async do
-  setTrace (← computeTrace (TextFilePath.mk path))
-  return path
+def inputTextFile (path : FilePath) : SpawnM (BuildJob FilePath) :=
+  Job.async <| (path, ·) <$> computeTrace (TextFilePath.mk path)
 
 /--
 A build job for file that is expected to already exist  (e.g., a data blob or source file).
@@ -299,7 +271,7 @@ Any byte difference in a binary file will trigger a rebuild of its dependents.
 In contrast, text file traces have normalized line endings to avoid unnecessary
 rebuilds across platforms.
 -/
-@[inline] def inputFile (path : FilePath) (text : Bool) : SpawnM (Job FilePath) :=
+@[inline] def inputFile (path : FilePath) (text : Bool) : SpawnM (BuildJob FilePath) :=
   if text then inputTextFile path else inputBinFile path
 
 /--
@@ -315,115 +287,99 @@ be `weakArgs` to avoid build artifact incompatibility between systems
 (i.e., a change in the file path should not cause a rebuild).
 
 You can add more components to the trace via `extraDepTrace`,
-which will be computed in the resulting `Job` before building.
+which will be computed in the resulting `BuildJob` before building.
 -/
 @[inline] def buildO
-  (oFile : FilePath) (srcJob : Job FilePath)
+  (oFile : FilePath) (srcJob : BuildJob FilePath)
   (weakArgs traceArgs : Array String := #[]) (compiler : FilePath := "cc")
   (extraDepTrace : JobM _ := pure BuildTrace.nil)
-: SpawnM (Job FilePath) :=
-  srcJob.mapM fun srcFile => do
-    addPlatformTrace -- object files are platform-dependent artifacts
-    addPureTrace traceArgs
-    addTrace (← extraDepTrace)
-    buildFileUnlessUpToDate' oFile do
-      compileO oFile srcFile (weakArgs ++ traceArgs) compiler
-    return oFile
+: SpawnM (BuildJob FilePath) :=
+  let extraDepTrace :=
+    return (← extraDepTrace).mix <| (pureHash traceArgs).mix platformTrace
+  buildFileAfterDep oFile srcJob (extraDepTrace := extraDepTrace) fun srcFile => do
+    compileO oFile srcFile (weakArgs ++ traceArgs) compiler
 
-/--
-Build an object file from a source fie job (i.e, a `lean -c` output)=
-using the Lean toolchain's C compiler.
--/
-def buildLeanO
-  (oFile : FilePath) (srcJob : Job FilePath)
+/-- Build an object file from a source fie job (i.e, a `lean -c` output) using `leanc`. -/
+@[inline] def buildLeanO
+  (oFile : FilePath) (srcJob : BuildJob FilePath)
   (weakArgs traceArgs : Array String := #[])
-: SpawnM (Job FilePath) :=
-  srcJob.mapM fun srcFile => do
-    addLeanTrace
-    addPureTrace traceArgs
-    addPlatformTrace -- object files are platform-dependent artifacts
-    buildFileUnlessUpToDate' oFile do
-      let lean ← getLeanInstall
-      compileO oFile srcFile (lean.ccFlags ++ weakArgs ++ traceArgs) lean.cc
-    return oFile
+: SpawnM (BuildJob FilePath) :=
+  let extraDepTrace :=
+    return (← getLeanTrace).mix <| (pureHash traceArgs).mix platformTrace
+  buildFileAfterDep oFile srcJob (extraDepTrace := extraDepTrace) fun srcFile => do
+    let lean ← getLeanInstall
+    compileO oFile srcFile (lean.ccFlags ++ weakArgs ++ traceArgs) lean.cc
 
-/-- Build a static library from object file jobs using the Lean toolchain's `ar`. -/
+/-- Build a static library from object file jobs using the `ar` packaged with Lean. -/
 def buildStaticLib
-  (libFile : FilePath) (oFileJobs : Array (Job FilePath))
-: SpawnM (Job FilePath) :=
-  (Job.collectArray oFileJobs).mapM fun oFiles => do
-    buildFileUnlessUpToDate' libFile do
-      compileStaticLib libFile oFiles (← getLeanAr)
-    return libFile
+  (libFile : FilePath) (oFileJobs : Array (BuildJob FilePath))
+: SpawnM (BuildJob FilePath) :=
+  buildFileAfterDepArray libFile oFileJobs fun oFiles => do
+    compileStaticLib libFile oFiles (← getLeanAr)
 
 /--
 Build a shared library by linking the results of `linkJobs`
 using the Lean toolchain's C compiler.
 -/
 def buildLeanSharedLib
-  (libFile : FilePath) (linkJobs : Array (Job FilePath))
+  (libFile : FilePath) (linkJobs : Array (BuildJob FilePath))
   (weakArgs traceArgs : Array String := #[])
-: SpawnM (Job FilePath) :=
-  (Job.collectArray linkJobs).mapM fun links => do
-    addLeanTrace
-    addPureTrace traceArgs
-    addPlatformTrace -- shared libraries are platform-dependent artifacts
-    buildFileUnlessUpToDate' libFile do
-      let lean ← getLeanInstall
-      let args := links.map toString ++ weakArgs ++ traceArgs ++ lean.ccLinkSharedFlags
-      compileSharedLib libFile args lean.cc
-    return libFile
+: SpawnM (BuildJob FilePath) :=
+  let extraDepTrace :=
+    return (← getLeanTrace).mix <| (pureHash traceArgs).mix platformTrace
+  buildFileAfterDepArray libFile linkJobs (extraDepTrace := extraDepTrace) fun links => do
+    let lean ← getLeanInstall
+    let args := links.map toString ++ weakArgs ++ traceArgs ++ lean.ccLinkSharedFlags
+    compileSharedLib libFile args lean.cc
+
 
 /--
 Build an executable by linking the results of `linkJobs`
 using the Lean toolchain's linker.
 -/
 def buildLeanExe
-  (exeFile : FilePath) (linkJobs : Array (Job FilePath))
+  (exeFile : FilePath) (linkJobs : Array (BuildJob FilePath))
   (weakArgs traceArgs : Array String := #[]) (sharedLean : Bool := false)
-: SpawnM (Job FilePath) :=
-  (Job.collectArray linkJobs).mapM fun links => do
-    addLeanTrace
-    addPureTrace traceArgs
-    addPlatformTrace -- executables are platform-dependent artifacts
-    buildFileUnlessUpToDate' exeFile do
-      let lean ← getLeanInstall
-      let args := weakArgs ++ traceArgs ++ lean.ccLinkFlags sharedLean
-      compileExe exeFile links args lean.cc
-    return exeFile
+: SpawnM (BuildJob FilePath) :=
+  let extraDepTrace :=
+    return (← getLeanTrace).mix <|
+      (pureHash sharedLean).mix <| (pureHash traceArgs).mix platformTrace
+  buildFileAfterDepArray exeFile linkJobs (extraDepTrace := extraDepTrace) fun links => do
+    let lean ← getLeanInstall
+    let args := weakArgs ++ traceArgs ++ lean.ccLinkFlags sharedLean
+    compileExe exeFile links args lean.cc
 
 /--
 Build a shared library from a static library using `leanc`
 using the Lean toolchain's linker.
 -/
 def buildLeanSharedLibOfStatic
-  (staticLibJob : Job FilePath)
+  (staticLibJob : BuildJob FilePath)
   (weakArgs traceArgs : Array String := #[])
-: SpawnM (Job FilePath) :=
-  staticLibJob.mapM fun staticLib => do
-    addLeanTrace
-    addPureTrace traceArgs
-    addPlatformTrace -- shared libraries are platform-dependent artifacts
+: SpawnM (BuildJob FilePath) :=
+  staticLibJob.bindSync fun staticLib staticTrace => do
     let dynlib := staticLib.withExtension sharedLibExt
-    buildFileUnlessUpToDate' dynlib do
-      let lean ← getLeanInstall
-      let baseArgs :=
-        if System.Platform.isOSX then
-          #[s!"-Wl,-force_load,{staticLib}"]
-        else
-          #["-Wl,--whole-archive", staticLib.toString, "-Wl,--no-whole-archive"]
-      let args := baseArgs ++ weakArgs ++ traceArgs ++ lean.ccLinkSharedFlags
+    let baseArgs :=
+      if System.Platform.isOSX then
+        #[s!"-Wl,-force_load,{staticLib}"]
+      else
+        #["-Wl,--whole-archive", staticLib.toString, "-Wl,--no-whole-archive"]
+    let lean ← getLeanInstall
+    let depTrace := staticTrace.mix <|
+      (← getLeanTrace).mix <| (pureHash traceArgs).mix <| platformTrace
+    let args := baseArgs ++ weakArgs ++ traceArgs ++ lean.ccLinkSharedFlags
+    let trace ← buildFileUnlessUpToDate dynlib depTrace do
       compileSharedLib dynlib args lean.cc
-    return dynlib
+    return (dynlib, trace)
 
 /-- Construct a `Dynlib` object for a shared library target. -/
-def computeDynlibOfShared (sharedLibTarget : Job FilePath) : SpawnM (Job Dynlib) :=
-  sharedLibTarget.mapM fun sharedLib => do
+def computeDynlibOfShared (sharedLibTarget : BuildJob FilePath) : SpawnM (BuildJob Dynlib) :=
+  sharedLibTarget.bindSync fun sharedLib trace => do
     if let some stem := sharedLib.fileStem then
       if Platform.isWindows then
-        return {path := sharedLib, name := stem}
+        return ({path := sharedLib, name := stem}, trace)
       else if stem.startsWith "lib" then
-        return {path := sharedLib, name := stem.drop 3}
+        return ({path := sharedLib, name := stem.drop 3}, trace)
       else
         error s!"shared library `{sharedLib}` does not start with `lib`; this is not supported on Unix"
     else

src/lake/Lake/Build/Executable.lean

@@ -13,7 +13,7 @@ open System (FilePath)
 The build function definition for a Lean executable.
 -/
 
-def LeanExe.recBuildExe (self : LeanExe) : FetchM (Job FilePath) :=
+def LeanExe.recBuildExe (self : LeanExe) : FetchM (BuildJob FilePath) :=
   withRegisterJob s!"{self.name}" do
   /-
   Remark: We must build the root before we fetch the transitive imports

src/lake/Lake/Build/Facets.lean

@@ -53,7 +53,7 @@ The facet which builds all of a module's dependencies
 Returns the list of shared libraries to load along with their search path.
 -/
 abbrev Module.depsFacet := `deps
-module_data deps : Job (SearchPath × Array FilePath)
+module_data deps : BuildJob (SearchPath × Array FilePath)
 
 /--
 The core build facet of a Lean file.
@@ -62,42 +62,42 @@ of the module (i.e., `olean`, `ilean`, `c`).
 Its trace just includes its dependencies.
 -/
 abbrev Module.leanArtsFacet := `leanArts
-module_data leanArts : Job Unit
+module_data leanArts : BuildJob Unit
 
 /-- The `olean` file produced by `lean`. -/
 abbrev Module.oleanFacet := `olean
-module_data olean : Job FilePath
+module_data olean : BuildJob FilePath
 
 /-- The `ilean` file produced by `lean`. -/
 abbrev Module.ileanFacet := `ilean
-module_data ilean : Job FilePath
+module_data ilean : BuildJob FilePath
 
 /-- The C file built from the Lean file via `lean`. -/
 abbrev Module.cFacet := `c
-module_data c : Job FilePath
+module_data c : BuildJob FilePath
 
 /-- The LLVM BC file built from the Lean file via `lean`. -/
 abbrev Module.bcFacet := `bc
-module_data bc : Job FilePath
+module_data bc : BuildJob FilePath
 
 /--
 The object file `.c.o` built from `c`.
 On Windows, this is `c.o.noexport`, on other systems it is `c.o.export`).
 -/
 abbrev Module.coFacet := `c.o
-module_data c.o : Job FilePath
+module_data c.o : BuildJob FilePath
 
 /-- The object file `.c.o.export` built from `c` (with `-DLEAN_EXPORTING`). -/
 abbrev Module.coExportFacet := `c.o.export
-module_data c.o.export : Job FilePath
+module_data c.o.export : BuildJob FilePath
 
 /-- The object file `.c.o.noexport` built from `c` (without `-DLEAN_EXPORTING`). -/
 abbrev Module.coNoExportFacet := `c.o.noexport
-module_data c.o.noexport : Job FilePath
+module_data c.o.noexport : BuildJob FilePath
 
 /-- The object file `.bc.o` built from `bc`. -/
 abbrev Module.bcoFacet := `bc.o
-module_data bc.o : Job FilePath
+module_data bc.o : BuildJob FilePath
 
 /--
 The object file built from `c`/`bc`.
@@ -105,15 +105,15 @@ On Windows with the C backend, no Lean symbols are exported.
 On every other configuration, symbols are exported.
 -/
 abbrev Module.oFacet := `o
-module_data o : Job FilePath
+module_data o : BuildJob FilePath
 
 /-- The object file built from `c`/`bc` (with Lean symbols exported). -/
 abbrev Module.oExportFacet := `o.export
-module_data o.export : Job FilePath
+module_data o.export : BuildJob FilePath
 
 /-- The object file built from `c`/`bc` (without Lean symbols exported). -/
 abbrev Module.oNoExportFacet := `o.noexport
-module_data o.noexport : Job FilePath
+module_data o.noexport : BuildJob FilePath
 
 
 /-! ## Package Facets -/
@@ -123,35 +123,35 @@ A package's *optional* cached build archive (e.g., from Reservoir or GitHub).
 Will NOT cause the whole build to fail if the archive cannot be fetched.
 -/
 abbrev Package.optBuildCacheFacet := `optCache
-package_data optCache : Job Bool
+package_data optCache : BuildJob Bool
 
 /--
 A package's cached build archive (e.g., from Reservoir or GitHub).
 Will cause the whole build to fail if the archive cannot be fetched.
 -/
 abbrev Package.buildCacheFacet := `cache
-package_data cache : Job Unit
+package_data cache : BuildJob Unit
 
 /--
 A package's *optional* build archive from Reservoir.
 Will NOT cause the whole build to fail if the barrel cannot be fetched.
 -/
 abbrev Package.optReservoirBarrelFacet := `optBarrel
-package_data optBarrel : Job Bool
+package_data optBarrel : BuildJob Bool
 
 /--
 A package's Reservoir build archive from Reservoir.
 Will cause the whole build to fail if the barrel cannot be fetched.
 -/
 abbrev Package.reservoirBarrelFacet := `barrel
-package_data barrel : Job Unit
+package_data barrel : BuildJob Unit
 
 /--
 A package's *optional* build archive from a GitHub release.
 Will NOT cause the whole build to fail if the release cannot be fetched.
 -/
 abbrev Package.optGitHubReleaseFacet := `optRelease
-package_data optRelease : Job Bool
+package_data optRelease : BuildJob Bool
 
 @[deprecated optGitHubReleaseFacet (since := "2024-09-27")]
 abbrev Package.optReleaseFacet := optGitHubReleaseFacet
@@ -161,49 +161,49 @@ A package's build archive from a GitHub release.
 Will cause the whole build to fail if the release cannot be fetched.
 -/
 abbrev Package.gitHubReleaseFacet := `release
-package_data release : Job Unit
+package_data release : BuildJob Unit
 
 @[deprecated gitHubReleaseFacet (since := "2024-09-27")]
 abbrev Package.releaseFacet := gitHubReleaseFacet
 
 /-- A package's `extraDepTargets` mixed with its transitive dependencies'. -/
 abbrev Package.extraDepFacet := `extraDep
-package_data extraDep : Job Unit
+package_data extraDep : BuildJob Unit
 
 /-! ## Target Facets -/
 
 /-- A Lean library's Lean artifacts (i.e., `olean`, `ilean`, `c`). -/
 abbrev LeanLib.leanArtsFacet := `leanArts
-library_data leanArts : Job Unit
+library_data leanArts : BuildJob Unit
 
 /-- A Lean library's static artifact. -/
 abbrev LeanLib.staticFacet := `static
-library_data static : Job FilePath
+library_data static : BuildJob FilePath
 
 /-- A Lean library's static artifact (with exported symbols). -/
 abbrev LeanLib.staticExportFacet := `static.export
-library_data static.export : Job FilePath
+library_data static.export : BuildJob FilePath
 
 /-- A Lean library's shared artifact. -/
 abbrev LeanLib.sharedFacet := `shared
-library_data shared : Job FilePath
+library_data shared : BuildJob FilePath
 
 /-- A Lean library's `extraDepTargets` mixed with its package's. -/
 abbrev LeanLib.extraDepFacet := `extraDep
-library_data extraDep : Job Unit
+library_data extraDep : BuildJob Unit
 
 /-- A Lean binary executable. -/
 abbrev LeanExe.exeFacet := `leanExe
-target_data leanExe : Job FilePath
+target_data leanExe : BuildJob FilePath
 
 /-- A external library's static binary. -/
 abbrev ExternLib.staticFacet := `externLib.static
-target_data externLib.static : Job FilePath
+target_data externLib.static : BuildJob FilePath
 
 /-- A external library's shared binary. -/
 abbrev ExternLib.sharedFacet := `externLib.shared
-target_data externLib.shared : Job FilePath
+target_data externLib.shared : BuildJob FilePath
 
 /-- A external library's dynlib. -/
 abbrev ExternLib.dynlibFacet := `externLib.dynlib
-target_data externLib.dynlib : Job Dynlib
+target_data externLib.dynlib : BuildJob Dynlib

src/lake/Lake/Build/Imports.lean

@@ -19,24 +19,26 @@ Used by `lake setup-file` to build modules for the Lean server and
 by `lake lean` to build the imports of a file.
 Returns the set of module dynlibs built (so they can be loaded by Lean).
 -/
-def buildImportsAndDeps (leanFile : FilePath) (imports : Array Module) : FetchM (Job (Array FilePath)) := do
+def buildImportsAndDeps (leanFile : FilePath) (imports : Array Module) : FetchM (BuildJob (Array FilePath)) := do
   withRegisterJob s!"imports ({leanFile})" do
   if imports.isEmpty then
     -- build the package's (and its dependencies') `extraDepTarget`
     (← getRootPackage).extraDep.fetch <&> (·.map fun _ => #[])
   else
     -- build local imports from list
-    let modJob := Job.mixArray <| ← imports.mapM (·.olean.fetch)
+    let modJob ← BuildJob.mixArray <| ← imports.mapM (·.olean.fetch)
     let precompileImports ← computePrecompileImportsAux leanFile imports
     let pkgs := precompileImports.foldl (·.insert ·.pkg) OrdPackageSet.empty |>.toArray
-    let externLibJob := Job.collectArray <| ←
+    let externLibJob ← BuildJob.collectArray <| ←
       pkgs.flatMapM (·.externLibs.mapM (·.dynlib.fetch))
-    let precompileJob := Job.collectArray <| ←
+    let precompileJob ← BuildJob.collectArray <| ←
       precompileImports.mapM (·.dynlib.fetch)
     let job ←
-      modJob.bindM fun _ =>
-      precompileJob.bindM fun modLibs =>
-      externLibJob.mapM fun externLibs => do
+      modJob.bindAsync fun _ modTrace =>
+      precompileJob.bindAsync fun modLibs modLibTrace =>
+      externLibJob.bindSync fun externLibs externTrace => do
         -- NOTE: Lean wants the external library symbols before module symbols
-        return (externLibs ++ modLibs).map (·.path)
+        let libs := (externLibs ++ modLibs).map (·.path)
+        let trace := modTrace.mix modLibTrace |>.mix externTrace
+        return (libs, trace)
     return job

src/lake/Lake/Build/Index.lean

@@ -25,20 +25,20 @@ Converts a conveniently-typed target facet build function into its
 dynamically-typed equivalent.
 -/
 @[macro_inline] def mkTargetFacetBuild
-  (facet : Name) (build : FetchM (Job α))
-  [h : FamilyOut TargetData facet (Job α)]
+  (facet : Name) (build : FetchM (BuildJob α))
+  [h : FamilyOut TargetData facet (BuildJob α)]
 : FetchM (TargetData facet) :=
   cast (by rw [← h.family_key_eq_type]) build
 
-def ExternLib.recBuildStatic (lib : ExternLib) : FetchM (Job FilePath) :=
+def ExternLib.recBuildStatic (lib : ExternLib) : FetchM (BuildJob FilePath) :=
   withRegisterJob s!"{lib.staticTargetName.toString}:static" do
   lib.config.getJob <$> fetch (lib.pkg.target lib.staticTargetName)
 
-def ExternLib.recBuildShared (lib : ExternLib) : FetchM (Job FilePath) :=
+def ExternLib.recBuildShared (lib : ExternLib) : FetchM (BuildJob FilePath) :=
   withRegisterJob s!"{lib.staticTargetName.toString}:shared" do
   buildLeanSharedLibOfStatic (← lib.static.fetch) lib.linkArgs
 
-def ExternLib.recComputeDynlib (lib : ExternLib) : FetchM (Job Dynlib) := do
+def ExternLib.recComputeDynlib (lib : ExternLib) : FetchM (BuildJob Dynlib) := do
   withRegisterJob s!"{lib.staticTargetName.toString}:dynlib" do
   computeDynlibOfShared (← lib.shared.fetch)
 

src/lake/Lake/Build/Info.lean

@@ -137,7 +137,7 @@ module_data lean.precompileImports : Array Module
 
 /-- Shared library for `--load-dynlib`. -/
 abbrev Module.dynlibFacet := `dynlib
-module_data dynlib : Job Dynlib
+module_data dynlib : BuildJob Dynlib
 
 /-- A Lean library's Lean modules. -/
 abbrev LeanLib.modulesFacet := `modules

src/lake/Lake/Build/Job.lean

@@ -4,40 +4,75 @@ Released under Apache 2.0 license as described in the file LICENSE.
 Authors: Mac Malone
 -/
 prelude
+import Lake.Util.Task
 import Lake.Build.Basic
 
 open System
 
 namespace Lake
 
+/-- Information on what this job did. -/
+inductive JobAction
+/-- No information about this job's action is available. -/
+| unknown
+/-- Tried to replay a cached build action (set by `buildFileUnlessUpToDate`) -/
+| replay
+/-- Tried to fetch a build from a store (can be set by `buildUnlessUpToDate?`) -/
+| fetch
+/-- Tried to perform a build action (set by `buildUnlessUpToDate?`) -/
+| build
+deriving Inhabited, Repr, DecidableEq, Ord
+
+instance : LT JobAction := ltOfOrd
+instance : LE JobAction := leOfOrd
+instance : Min JobAction := minOfLe
+instance : Max JobAction := maxOfLe
+
+@[inline] def JobAction.merge (a b : JobAction) : JobAction :=
+  max a b
+
 def JobAction.verb (failed : Bool) : JobAction → String
 | .unknown => if failed then "Running" else "Ran"
 | .replay => if failed then "Replaying" else "Replayed"
 | .fetch => if failed then "Fetching" else "Fetched"
 | .build => if failed then "Building" else "Built"
 
+/-- Mutable state of a Lake job. -/
+structure JobState where
+  /-- The job's log. -/
+  log : Log := {}
+  /-- Tracks whether this job performed any significant build action. -/
+  action : JobAction := .unknown
+  deriving Inhabited
+
 /--
 Resets the job state after a checkpoint (e.g., registering the job).
 Preserves state that downstream jobs want to depend on while resetting
 job-local state that should not be inherited by downstream jobs.
 -/
-@[inline] def JobState.renew (s : JobState) : JobState where
-  trace := s.trace
+@[inline] def JobState.renew (_ : JobState) : JobState where
+  log := {}
+  action := .unknown
 
 def JobState.merge (a b : JobState) : JobState where
   log := a.log ++ b.log
   action := a.action.merge b.action
-  trace := mixTrace a.trace b.trace
 
 @[inline] def JobState.modifyLog (f : Log → Log) (s : JobState) :=
   {s with log := f s.log}
 
+/-- The result of a Lake job. -/
+abbrev JobResult α := EResult Log.Pos JobState α
+
 /-- Add log entries to the beginning of the job's log. -/
 def JobResult.prependLog (log : Log) (self : JobResult α) : JobResult α :=
   match self with
   | .ok a s => .ok a <| s.modifyLog (log ++ ·)
   | .error e s => .error ⟨log.size + e.val⟩ <| s.modifyLog (log ++ ·)
 
+/-- The `Task` of a Lake job. -/
+abbrev JobTask α := BaseIOTask (JobResult α)
+
 /--
 The monad of asynchronous Lake jobs.
 
@@ -66,42 +101,39 @@ instance : MonadLift LogIO JobM := ⟨ELogT.takeAndRun⟩
 @[inline] def updateAction (action : JobAction) : JobM PUnit :=
   modify fun s => {s with action := s.action.merge action}
 
-/-- Returns the current job's build trace. -/
-@[inline] def getTrace : JobM BuildTrace :=
-  (·.trace) <$> get
-
-/-- Sets the current job's build trace. -/
-@[inline] def setTrace (trace : BuildTrace) : JobM PUnit :=
-  modify fun s => {s with trace := trace}
-
-/-- Mix a trace into the current job's build trace. -/
-@[inline] def addTrace (trace : BuildTrace) : JobM PUnit :=
-  modify fun s => {s with trace := s.trace.mix trace}
-
-/-- Returns the current job's build trace and removes it from the state. -/
-@[inline] def takeTrace : JobM BuildTrace :=
-  modifyGet fun s => (s.trace, {s with trace := nilTrace})
-
 /-- The monad used to spawn asynchronous Lake build jobs. Lifts into `FetchM`. -/
-abbrev SpawnM := BuildT <| ReaderT BuildTrace <| BaseIO
-
-@[inline] def JobM.runSpawnM (x : SpawnM α) : JobM α := fun ctx s =>
-  return .ok (← x ctx s.trace) s
-
-instance : MonadLift SpawnM JobM := ⟨JobM.runSpawnM⟩
+abbrev SpawnM := BuildT BaseIO
 
 /-- The monad used to spawn asynchronous Lake build jobs. **Replaced by `SpawnM`.** -/
 @[deprecated SpawnM (since := "2024-05-21")] abbrev SchedulerM := SpawnM
 
-@[inline] private unsafe def Job.toOpaqueImpl (job : Job α) : OpaqueJob :=
-  unsafeCast job
+/-- A Lake job. -/
+abbrev Job (α : Type u)  := JobCore (JobTask α)
 
-/-- Forget the value of a job. Implemented as a no-op cast. -/
-@[implemented_by toOpaqueImpl]
-def Job.toOpaque (job : Job α) : OpaqueJob :=
-  {job with task := job.task.map (·.map Opaque.mk)}
+structure BundledJobTask where
+  {Result : Type u}
+  task : JobTask Result
+  deriving Inhabited
 
-instance : CoeOut (Job α) OpaqueJob := ⟨.toOpaque⟩
+instance : CoeOut (JobTask α) BundledJobTask := ⟨.mk⟩
+
+hydrate_opaque_type OpaqueJobTask BundledJobTask
+
+abbrev OpaqueJob.Result (job : OpaqueJob) : Type :=
+  job.task.get.Result
+
+nonrec abbrev OpaqueJob.task (job : OpaqueJob) : JobTask job.Result :=
+  job.task.get.task
+
+abbrev OpaqueJob.ofJob (job : Job α) : OpaqueJob :=
+  {job with task := job.task}
+
+instance : CoeOut (Job α) OpaqueJob := ⟨.ofJob⟩
+
+abbrev OpaqueJob.toJob (job : OpaqueJob) : Job job.Result :=
+  {job with task := job.task}
+
+instance : CoeDep OpaqueJob job (Job job.Result) := ⟨job.toJob⟩
 
 namespace Job
 
@@ -120,9 +152,6 @@ instance [Inhabited α] : Inhabited (Job α) := ⟨pure default⟩
 @[inline] protected def nop (log : Log := {}) (caption := "") : Job Unit :=
   .pure () log caption
 
-@[inline] def nil : Job Unit :=
-  .pure ()
-
 /-- Sets the job's caption. -/
 @[inline] def setCaption (caption : String) (job : Job α) : Job α :=
   {job with caption}
@@ -134,24 +163,19 @@ instance [Inhabited α] : Inhabited (Job α) := ⟨pure default⟩
 @[inline] def mapResult
   (f : JobResult α → JobResult β) (self : Job α)
   (prio := Task.Priority.default) (sync := false)
-: Job β := {self with task := self.task.map f prio sync}
-
-@[inline] def mapOk
-  (f : α → JobState → JobResult β) (self : Job α)
-  (prio := Task.Priority.default) (sync := false)
 : Job β :=
-  self.mapResult (prio := prio) (sync := sync) fun
-    | .ok a s => f a s
-    | .error e s => .error e s
+  {self with task := self.task.map f prio sync}
 
 @[inline] def bindTask [Monad m]
   (f : JobTask α → m (JobTask β)) (self : Job α)
-: m (Job β) := return {self with task := ← f self.task}
+: m (Job β) :=
+  return {self with task := ← f self.task}
 
 @[inline] protected def map
   (f : α → β) (self : Job α)
   (prio := Task.Priority.default) (sync := false)
-: Job β := self.mapResult (·.map f) prio sync
+: Job β :=
+  self.mapResult (·.map f) prio sync
 
 instance : Functor Job where map := Job.map
 
@@ -191,194 +215,115 @@ Logs the job's log and throws if there was an error.
   | .error n {log, ..} => log.replay; throw n
   | .ok a {log, ..} => log.replay; pure a
 
-/-- Apply `f` asynchronously to the job's output. -/
-protected def mapM
+/--
+`let c ← a.bindSync b` asynchronously performs the action `b`
+after the job `a` completes.
+-/
+@[inline] protected def bindSync
   (self : Job α) (f : α → JobM β)
   (prio := Task.Priority.default) (sync := false)
-: SpawnM (Job β) :=
-  fun ctx trace => do
-  self.bindTask fun task => do
+: SpawnM (Job β) := fun ctx => self.bindTask fun task => do
   BaseIO.mapTask (t := task) (prio := prio) (sync := sync) fun
-    | .ok a s =>
-      let trace := mixTrace trace s.trace
-      withLoggedIO (f a) ctx {s with trace}
-    | .error n s => return .error n s
-
-@[deprecated Job.mapM (since := "2024-12-06")]
-protected abbrev bindSync
-  (self : Job α) (f : α → JobM β)
-  (prio := Task.Priority.default) (sync := false)
-: SpawnM (Job β) := self.mapM f prio sync
+    | EResult.ok a s => (withLoggedIO (f a)) ctx s
+    | EResult.error n s => return .error n s
 
 /--
-Apply `f` asynchronously to the job's output
-and asynchronously await the resulting job.
+`let c ← a.bindAsync b` asynchronously performs the action `b`
+after the job `a` completes and then merges into the job produced by `b`.
 -/
-def bindM
-  (self : Job α) (f : α → JobM (Job β))
-  (prio := Task.Priority.default) (sync := false)
-: SpawnM (Job β) :=
-  fun ctx trace => do
-  self.bindTask fun task => do
-  BaseIO.bindTask task (prio := prio) (sync := sync) fun
-    | .ok a sa => do
-      let trace := mixTrace trace sa.trace
-      match (← withLoggedIO (f a) ctx {sa with trace}) with
-      | .ok job sa =>
-        return job.task.map (prio := prio) (sync := true) fun
-        | .ok b sb => .ok b {sa.merge sb with trace := sb.trace}
-        | .error e sb => .error ⟨sa.log.size + e.val⟩ {sa.merge sb with trace := sb.trace}
-      | .error e sa => return Task.pure (.error e sa)
-    | .error e sa => return Task.pure (.error e sa)
-
-@[deprecated bindM (since := "2024-12-06")]
-protected abbrev bindAsync
+@[inline] protected def bindAsync
   (self : Job α) (f : α → SpawnM (Job β))
   (prio := Task.Priority.default) (sync := false)
-: SpawnM (Job β) := self.bindM (fun a => f a) prio sync
-
-/--
-`a.zipWith f b` produces a new job `c` that applies `f` to the
-results of `a` and `b`. The job `c` errors if either `a` or `b` error.
--/
-@[inline] def zipResultWith
-  (f : JobResult α → JobResult β → JobResult γ) (self : Job α) (other : Job β)
-  (prio := Task.Priority.default) (sync := true)
-: Job γ := Job.ofTask $
-  self.task.bind (prio := prio) (sync := true) fun rx =>
-  other.task.map (prio := prio) (sync := sync) fun ry =>
-  f rx ry
+: SpawnM (Job β) := fun ctx => self.bindTask fun task => do
+  BaseIO.bindTask task (prio := prio) (sync := sync) fun
+    | .ok a sa => do
+      let job ← f a ctx
+      return job.task.map (prio := prio) (sync := true) fun
+      | EResult.ok a sb => .ok a (sa.merge sb)
+      | EResult.error n sb => .error ⟨sa.log.size + n.val⟩ (sa.merge sb)
+    | .error n l => return Task.pure (.error n l)
 
 /--
 `a.zipWith f b` produces a new job `c` that applies `f` to the
 results of `a` and `b`. The job `c` errors if either `a` or `b` error.
 -/
 @[inline] def zipWith
-  (f : α → β → γ) (self : Job α) (other : Job β)
-  (prio := Task.Priority.default) (sync := true)
-: Job γ :=
-  self.zipResultWith (other := other) (prio := prio) (sync := sync) fun
+  (f : α → β → γ) (x : Job α) (y : Job β)
+  (prio := Task.Priority.default) (sync := false)
+: Job γ := Job.ofTask $
+  x.task.bind (prio := prio) (sync := true) fun rx =>
+  y.task.map (prio := prio) (sync := sync) fun ry =>
+  match rx, ry with
   | .ok a sa, .ok b sb => .ok (f a b) (sa.merge sb)
-  | ra, rb => .error 0 (ra.state.merge rb.state)
-
-
-/-- Merges this job with another, discarding its output and trace. -/
-def add (self : Job α) (other : Job β) : Job α :=
-  self.zipResultWith (other := other) fun
-  | .ok a sa, .ok _ sb => .ok a (sa.merge sb)
-  | ra, rb => .error 0 {ra.state.merge rb.state with trace := ra.state.trace}
-
-/-- Merges this job with another, discarding both outputs. -/
-def mix (self : Job α) (other : Job β) : Job Unit :=
-  self.zipWith (fun _ _ => ()) other
-
-/-- Merge a `List` of jobs into one, discarding their outputs. -/
-def mixList (jobs : List (Job α)) : Job Unit :=
-  jobs.foldr (·.mix ·) nil
-
-/-- Merge an `Array` of jobs into one, discarding their outputs. -/
-def mixArray (jobs : Array (Job α)) : Job Unit :=
-  jobs.foldl (·.mix ·) nil
-
-/-- Merge a `List` of jobs into one, collecting their outputs into a `List`. -/
-def collectList (jobs : List (Job α)) : Job (List α) :=
-  jobs.foldr (zipWith List.cons) (pure [])
-
-/-- Merge an `Array` of jobs into one, collecting their outputs into an `Array`. -/
-def collectArray (jobs : Array (Job α)) : Job (Array α) :=
-  jobs.foldl (zipWith Array.push) (pure (Array.mkEmpty jobs.size))
+  | rx, ry => .error 0 (rx.state.merge ry.state)
 
 end Job
 
 /-- A Lake build job. -/
-abbrev BuildJob α := Job α
+abbrev BuildJob α := Job (α × BuildTrace)
 
 namespace BuildJob
 
-@[inline, deprecated "Obsolete." (since := "2024-12-06")]
-def mk (job : Job (α × BuildTrace)) : BuildJob α :=
-  job.mapOk fun (a, trace) s => .ok a {s with trace}
+@[inline] def mk (job : Job (α × BuildTrace)) : BuildJob α :=
+  job
 
-@[inline, deprecated "Obsolete." (since := "2024-12-06")]
-def ofJob (job : Job BuildTrace) : BuildJob Unit :=
-  job.mapOk fun trace s => .ok () {s with trace}
+@[inline] def ofJob (self : Job BuildTrace) : BuildJob Unit :=
+  mk <| ((), ·) <$> self
 
-abbrev toJob (self : BuildJob α) : Job α :=
+@[inline] def toJob (self : BuildJob α) : Job (α × BuildTrace) :=
   self
 
-@[deprecated Job.nil (since := "2024-12-06")]
-abbrev nil : BuildJob Unit :=
-  Job.pure ()
+@[inline] def nil : BuildJob Unit :=
+  mk <| pure ((), nilTrace)
 
-@[deprecated Job.map (since := "2024-12-06")]
-protected abbrev pure (a : α) : BuildJob α :=
-  Job.pure a
+@[inline] protected def pure (a : α) : BuildJob α :=
+  mk <| pure (a, nilTrace)
 
-instance : Pure BuildJob := ⟨Job.pure⟩
+instance : Pure BuildJob := ⟨BuildJob.pure⟩
 
-@[deprecated Job.map (since := "2024-12-06")]
-protected abbrev map (f : α → β) (self : BuildJob α) : BuildJob β :=
-  self.toJob.map f
+@[inline] protected def map (f : α → β) (self : BuildJob α) : BuildJob β :=
+  mk <| (fun (a,t) => (f a,t)) <$> self.toJob
 
 instance : Functor BuildJob where
-  map := Job.map
+  map := BuildJob.map
 
-@[inline, deprecated "Removed without replacement." (since := "2024-12-06")]
-def mapWithTrace (f : α → BuildTrace → β × BuildTrace) (self : BuildJob α) : BuildJob β :=
-  self.toJob.mapOk fun a s =>
-    let (b, trace) := f a s.trace
-    .ok b {s with trace}
+@[inline] def mapWithTrace (f : α → BuildTrace → β × BuildTrace) (self : BuildJob α) : BuildJob β :=
+  mk <| (fun (a,t) => f a t) <$> self.toJob
 
-@[inline, deprecated Job.mapM (since := "2024-12-06")]
-protected def bindSync
-  (self : BuildJob α) (f : α → BuildTrace → JobM (β × BuildTrace))
+@[inline] protected def bindSync
+  (self : BuildJob α) (f : α → BuildTrace → JobM β)
   (prio : Task.Priority := .default) (sync := false)
 : SpawnM (Job β) :=
-  self.toJob.mapM (prio := prio) (sync := sync) fun a => do
-    let (b, trace) ← f a (← getTrace)
-    setTrace trace
-    return b
+  self.toJob.bindSync (prio := prio) (sync := sync) fun (a, t) => f a t
 
-@[inline, deprecated Job.bindM (since := "2024-12-06")]
-protected def bindAsync
+@[inline] protected def bindAsync
   (self : BuildJob α) (f : α → BuildTrace → SpawnM (Job β))
   (prio : Task.Priority := .default) (sync := false)
  : SpawnM (Job β)  :=
-  self.toJob.bindM (prio := prio) (sync := sync) fun a => do
-    f a (← getTrace)
+  self.toJob.bindAsync (prio := prio) (sync := sync) fun (a, t) => f a t
 
-@[deprecated Job.wait? (since := "2024-12-06")]
-protected abbrev wait? (self : BuildJob α) : BaseIO (Option α) :=
-  self.toJob.wait?
+@[inline] protected def wait? (self : BuildJob α) : BaseIO (Option α) :=
+  (·.map (·.1)) <$> self.toJob.wait?
 
-@[deprecated Job.add (since := "2024-12-06")]
-abbrev add (self : BuildJob α) (other : BuildJob β) : BuildJob α :=
-  self.toJob.add other.toJob
+def add (self : BuildJob α) (other : BuildJob β) : BuildJob α :=
+  mk <| self.toJob.zipWith (fun a _ => a) other.toJob
 
-@[deprecated Job.mix (since := "2024-12-06")]
-abbrev mix (self : BuildJob α) (other : BuildJob β) : BuildJob Unit :=
-  self.toJob.mix other.toJob
+def mix (self : BuildJob α) (other : BuildJob β) : BuildJob Unit :=
+  mk <| self.toJob.zipWith (fun (_,t) (_,t') => ((), mixTrace t t')) other.toJob
 
-@[deprecated Job.mixList (since := "2024-12-06")]
-abbrev mixList (jobs : List (BuildJob α)) : Id (BuildJob Unit) :=
-  return Job.mixList jobs
+def mixList (jobs : List (BuildJob α)) : Id (BuildJob Unit) := ofJob $
+  jobs.foldr (·.toJob.zipWith (fun (_,t') t => mixTrace t t') ·) (pure nilTrace)
 
-@[deprecated Job.mixArray (since := "2024-12-06")]
-abbrev mixArray (jobs : Array (BuildJob α)) : Id (BuildJob Unit) :=
-  return Job.mixArray jobs
+def mixArray (jobs : Array (BuildJob α)) : Id (BuildJob Unit) := ofJob $
+  jobs.foldl (·.zipWith (fun t (_,t') => mixTrace t t') ·.toJob) (pure nilTrace)
 
-@[deprecated Job.zipWith (since := "2024-12-06")]
-abbrev zipWith
+def zipWith
   (f : α → β → γ) (self : BuildJob α) (other : BuildJob β)
 : BuildJob γ :=
-  self.toJob.zipWith f other.toJob
+  mk <| self.toJob.zipWith (fun (a,t) (b,t') => (f a b, mixTrace t t')) other.toJob
 
-@[deprecated Job.collectList (since := "2024-12-06")]
-abbrev collectList (jobs : List (BuildJob α)) : Id (BuildJob (List α)) :=
-  return Job.collectList jobs
+def collectList (jobs : List (BuildJob α)) : Id (BuildJob (List α)) :=
+  return jobs.foldr (zipWith List.cons) (pure [])
 
-@[deprecated Job.collectArray (since := "2024-12-06")]
-abbrev collectArray (jobs : Array (BuildJob α)) : Id (BuildJob (Array α)) :=
-  return Job.collectArray jobs
-
-attribute [deprecated Job (since := "2024-12-06")] BuildJob
+def collectArray (jobs : Array (BuildJob α)) : Id (BuildJob (Array α)) :=
+  return jobs.foldl (zipWith Array.push) (pure (Array.mkEmpty jobs.size))

src/lake/Lake/Build/Library.lean

@@ -44,9 +44,9 @@ def LeanLib.modulesFacetConfig : LibraryFacetConfig modulesFacet :=
   mkFacetConfig LeanLib.recCollectLocalModules
 
 protected def LeanLib.recBuildLean
-(self : LeanLib) : FetchM (Job Unit) := do
+(self : LeanLib) : FetchM (BuildJob Unit) := do
   let mods ← self.modules.fetch
-  mods.foldlM (init := Job.nil) fun job mod => do
+  mods.foldlM (init := BuildJob.nil) fun job mod => do
     return job.mix <| ← mod.leanArts.fetch
 
 /-- The `LibraryFacetConfig` for the builtin `leanArtsFacet`. -/
@@ -54,7 +54,7 @@ def LeanLib.leanArtsFacetConfig : LibraryFacetConfig leanArtsFacet :=
   mkFacetJobConfig LeanLib.recBuildLean
 
 @[specialize] protected def LeanLib.recBuildStatic
-(self : LeanLib) (shouldExport : Bool) : FetchM (Job FilePath) := do
+(self : LeanLib) (shouldExport : Bool) : FetchM (BuildJob FilePath) := do
   let suffix :=
     if (← getIsVerbose) then
       if shouldExport then " (with exports)" else " (without exports)"
@@ -79,7 +79,7 @@ def LeanLib.staticExportFacetConfig : LibraryFacetConfig staticExportFacet :=
 /-! ## Build Shared Lib -/
 
 protected def LeanLib.recBuildShared
-(self : LeanLib) : FetchM (Job FilePath) := do
+(self : LeanLib) : FetchM (BuildJob FilePath) := do
   withRegisterJob s!"{self.name}:shared" do
   let mods ← self.modules.fetch
   let oJobs ← mods.flatMapM fun mod =>
@@ -95,7 +95,7 @@ def LeanLib.sharedFacetConfig : LibraryFacetConfig sharedFacet :=
 /-! ## Build `extraDepTargets` -/
 
 /-- Build the `extraDepTargets` for the library and its package. -/
-def LeanLib.recBuildExtraDepTargets (self : LeanLib) : FetchM (Job Unit) := do
+def LeanLib.recBuildExtraDepTargets (self : LeanLib) : FetchM (BuildJob Unit) := do
   self.extraDepTargets.foldlM (init := ← self.pkg.extraDep.fetch) fun job target => do
     return job.mix <| ← self.pkg.fetchTargetJob target
 

src/lake/Lake/Build/Module.lean

@@ -19,9 +19,9 @@ namespace Lake
 
 /-- Compute library directories and build external library Jobs of the given packages. -/
 def recBuildExternDynlibs (pkgs : Array Package)
-: FetchM (Array (Job Dynlib) × Array FilePath) := do
+: FetchM (Array (BuildJob Dynlib) × Array FilePath) := do
   let mut libDirs := #[]
-  let mut jobs : Array (Job Dynlib) := #[]
+  let mut jobs : Array (BuildJob Dynlib) := #[]
   for pkg in pkgs do
     libDirs := libDirs.push pkg.nativeLibDir
     jobs := jobs.append <| ← pkg.externLibs.mapM (·.dynlib.fetch)
@@ -95,7 +95,7 @@ Recursively build a module's dependencies, including:
 * Shared libraries (e.g., `extern_lib` targets or precompiled modules)
 * `extraDepTargets` of its library
 -/
-def Module.recBuildDeps (mod : Module) : FetchM (Job (SearchPath × Array FilePath)) := ensureJob do
+def Module.recBuildDeps (mod : Module) : FetchM (BuildJob (SearchPath × Array FilePath)) := ensureJob do
   let extraDepJob ← mod.lib.extraDep.fetch
 
   /-
@@ -104,7 +104,7 @@ def Module.recBuildDeps (mod : Module) : FetchM (Job (SearchPath × Array FilePa
   will not kill this job before the direct imports are built.
   -/
   let directImports ← mod.imports.fetch
-  let importJob := Job.mixArray <| ← directImports.mapM fun imp => do
+  let importJob ← BuildJob.mixArray <| ← directImports.mapM fun imp => do
     if imp.name = mod.name then
       logError s!"{mod.leanFile}: module imports itself"
     imp.olean.fetch
@@ -114,18 +114,19 @@ def Module.recBuildDeps (mod : Module) : FetchM (Job (SearchPath × Array FilePa
   let pkgs := precompileImports.foldl (·.insert ·.pkg) OrdPackageSet.empty
   let pkgs := if mod.shouldPrecompile then pkgs.insert mod.pkg else pkgs
   let (externJobs, libDirs) ← recBuildExternDynlibs pkgs.toArray
-  let externDynlibsJob := Job.collectArray externJobs
-  let modDynlibsJob := Job.collectArray modLibJobs
+  let externDynlibsJob ← BuildJob.collectArray externJobs
+  let modDynlibsJob ← BuildJob.collectArray modLibJobs
 
-  extraDepJob.bindM fun _ => do
-  importJob.bindM fun _ => do
-  let depTrace ← takeTrace
-  modDynlibsJob.bindM fun modDynlibs => do
-  externDynlibsJob.mapM fun externDynlibs => do
-    match mod.platformIndependent with
-    | none => addTrace depTrace
-    | some false => addTrace depTrace; addPlatformTrace
-    | some true => setTrace depTrace
+  extraDepJob.bindAsync fun _ extraDepTrace => do
+  importJob.bindAsync fun _ importTrace => do
+  modDynlibsJob.bindAsync fun modDynlibs modLibTrace => do
+  return externDynlibsJob.mapWithTrace fun externDynlibs externTrace =>
+    let depTrace := extraDepTrace.mix <| importTrace
+    let depTrace :=
+      match mod.platformIndependent with
+      | none => depTrace.mix <| modLibTrace.mix <| externTrace
+      | some false => depTrace.mix <| modLibTrace.mix <| externTrace.mix <| platformTrace
+      | some true => depTrace
     /-
     Requirements:
     * Lean wants the external library symbols before module symbols.
@@ -136,7 +137,7 @@ def Module.recBuildDeps (mod : Module) : FetchM (Job (SearchPath × Array FilePa
     -/
     let dynlibPath := libDirs ++ externDynlibs.filterMap (·.dir?) |>.toList
     let dynlibs := externDynlibs.map (·.path) ++ modDynlibs.map (·.path)
-    return (dynlibPath, dynlibs)
+    ((dynlibPath, dynlibs), depTrace)
 
 /-- The `ModuleFacetConfig` for the builtin `depsFacet`. -/
 def Module.depsFacetConfig : ModuleFacetConfig depsFacet :=
@@ -163,19 +164,19 @@ Recursively build a Lean module.
 Fetch its dependencies and then elaborate the Lean source file, producing
 all possible artifacts (i.e., `.olean`, `ilean`, `.c`, and `.bc`).
 -/
-def Module.recBuildLean (mod : Module) : FetchM (Job Unit) := do
+def Module.recBuildLean (mod : Module) : FetchM (BuildJob Unit) := do
   withRegisterJob mod.name.toString do
-  (← mod.deps.fetch).mapM fun (dynlibPath, dynlibs) => do
-    addLeanTrace
-    addPureTrace mod.leanArgs
-    let srcTrace ← computeTrace (TextFilePath.mk mod.leanFile)
-    addTrace srcTrace
-    let upToDate ← buildUnlessUpToDate? (oldTrace := srcTrace.mtime) mod (← getTrace) mod.traceFile do
+  (← mod.deps.fetch).bindSync fun (dynlibPath, dynlibs) depTrace => do
+    let argTrace : BuildTrace := pureHash mod.leanArgs
+    let srcTrace : BuildTrace ← computeTrace { path := mod.leanFile : TextFilePath }
+    let modTrace := (← getLeanTrace).mix <| argTrace.mix <| srcTrace.mix depTrace
+    let upToDate ← buildUnlessUpToDate? (oldTrace := srcTrace.mtime) mod modTrace mod.traceFile do
       compileLeanModule mod.leanFile mod.oleanFile mod.ileanFile mod.cFile mod.bcFile?
         (← getLeanPath) mod.rootDir dynlibs dynlibPath (mod.weakLeanArgs ++ mod.leanArgs) (← getLean)
       mod.clearOutputHashes
     unless upToDate && (← getTrustHash) do
       mod.cacheOutputHashes
+    return ((), depTrace)
 
 /-- The `ModuleFacetConfig` for the builtin `leanArtsFacet`. -/
 def Module.leanArtsFacetConfig : ModuleFacetConfig leanArtsFacet :=
@@ -184,48 +185,34 @@ def Module.leanArtsFacetConfig : ModuleFacetConfig leanArtsFacet :=
 /-- The `ModuleFacetConfig` for the builtin `oleanFacet`. -/
 def Module.oleanFacetConfig : ModuleFacetConfig oleanFacet :=
   mkFacetJobConfig fun mod => do
-    (← mod.leanArts.fetch).mapM fun _ => do
-      addTrace (← fetchFileTrace mod.oleanFile)
-      return mod.oleanFile
+    (← mod.leanArts.fetch).bindSync fun _ depTrace =>
+      return (mod.oleanFile, mixTrace (← fetchFileTrace mod.oleanFile) depTrace)
 
 /-- The `ModuleFacetConfig` for the builtin `ileanFacet`. -/
 def Module.ileanFacetConfig : ModuleFacetConfig ileanFacet :=
   mkFacetJobConfig fun mod => do
-    (← mod.leanArts.fetch).mapM fun _ => do
-      addTrace (← fetchFileTrace mod.ileanFile)
-      return mod.ileanFile
+    (← mod.leanArts.fetch).bindSync fun _ depTrace =>
+      return (mod.ileanFile, mixTrace (← fetchFileTrace mod.ileanFile) depTrace)
 
 /-- The `ModuleFacetConfig` for the builtin `cFacet`. -/
 def Module.cFacetConfig : ModuleFacetConfig cFacet :=
   mkFacetJobConfig fun mod => do
-    (← mod.leanArts.fetch).mapM fun _ => do
-      /-
-      Avoid recompiling unchanged C files
-      C files are assumed to only depend on their content
-      and not transitively on their inputs (e.g., module sources).
-      Lean also produces LF-only C files, so no line ending normalization.
-      -/
-      setTrace (← fetchFileTrace mod.cFile)
-      addLeanTrace -- Lean C files include `lean/lean.h`
-      return mod.cFile
+    (← mod.leanArts.fetch).bindSync fun _ _ =>
+      -- do content-aware hashing so that we avoid recompiling unchanged C files
+      return (mod.cFile, ← fetchFileTrace mod.cFile)
 
 /-- The `ModuleFacetConfig` for the builtin `bcFacet`. -/
 def Module.bcFacetConfig : ModuleFacetConfig bcFacet :=
   mkFacetJobConfig fun mod => do
-    (← mod.leanArts.fetch).mapM fun _ => do
-      /-
-      Avoid recompiling unchanged bitcode files
-      Bitcode files are assumed to only depend on their content
-      and not transitively on their inputs (e.g., module sources)
-      -/
-      setTrace (← fetchFileTrace mod.bcFile)
-      return mod.bcFile
+    (← mod.leanArts.fetch).bindSync fun _ _ =>
+      -- do content-aware hashing so that we avoid recompiling unchanged bitcode files
+      return (mod.bcFile, ← fetchFileTrace mod.bcFile)
 
 /--
 Recursively build the module's object file from its C file produced by `lean`
 with `-DLEAN_EXPORTING` set, which exports Lean symbols defined within the C files.
 -/
-def Module.recBuildLeanCToOExport (self : Module) : FetchM (Job FilePath) := do
+def Module.recBuildLeanCToOExport (self : Module) : FetchM (BuildJob FilePath) := do
   let suffix := if (← getIsVerbose) then " (with exports)" else ""
   withRegisterJob s!"{self.name}:c.o{suffix}" do
   -- TODO: add option to pass a target triplet for cross compilation
@@ -240,7 +227,7 @@ def Module.coExportFacetConfig : ModuleFacetConfig coExportFacet :=
 Recursively build the module's object file from its C file produced by `lean`.
 This version does not export any Lean symbols.
 -/
-def Module.recBuildLeanCToONoExport (self : Module) : FetchM (Job FilePath) := do
+def Module.recBuildLeanCToONoExport (self : Module) : FetchM (BuildJob FilePath) := do
   let suffix := if (← getIsVerbose) then " (without exports)" else ""
   withRegisterJob s!"{self.name}:c.o{suffix}" do
   -- TODO: add option to pass a target triplet for cross compilation
@@ -256,7 +243,7 @@ def Module.coFacetConfig : ModuleFacetConfig coFacet :=
     if Platform.isWindows then mod.coNoExport.fetch else mod.coExport.fetch
 
 /-- Recursively build the module's object file from its bitcode file produced by `lean`. -/
-def Module.recBuildLeanBcToO (self : Module) : FetchM (Job FilePath) := do
+def Module.recBuildLeanBcToO (self : Module) : FetchM (BuildJob FilePath) := do
   withRegisterJob s!"{self.name}:bc.o" do
   -- TODO: add option to pass a target triplet for cross compilation
   buildLeanO self.bcoFile (← self.bc.fetch) self.weakLeancArgs self.leancArgs
@@ -286,9 +273,9 @@ def Module.oFacetConfig : ModuleFacetConfig oFacet :=
     | .default | .c => mod.co.fetch
     | .llvm => mod.bco.fetch
 
--- TODO: Return `Job OrdModuleSet × OrdPackageSet` or `OrdRBSet Dynlib`
+-- TODO: Return `BuildJob OrdModuleSet × OrdPackageSet` or `OrdRBSet Dynlib`
 /-- Recursively build the shared library of a module (e.g., for `--load-dynlib`). -/
-def Module.recBuildDynlib (mod : Module) : FetchM (Job Dynlib) :=
+def Module.recBuildDynlib (mod : Module) : FetchM (BuildJob Dynlib) :=
   withRegisterJob s!"{mod.name}:dynlib" do
 
   -- Compute dependencies
@@ -300,27 +287,27 @@ def Module.recBuildDynlib (mod : Module) : FetchM (Job Dynlib) :=
   let linkJobs ← mod.nativeFacets true |>.mapM (fetch <| mod.facet ·.name)
 
   -- Collect Jobs
-  let linksJob := Job.collectArray linkJobs
-  let modDynlibsJob := Job.collectArray modJobs
-  let externDynlibsJob := Job.collectArray externJobs
+  let linksJob ← BuildJob.collectArray linkJobs
+  let modDynlibsJob ← BuildJob.collectArray modJobs
+  let externDynlibsJob ← BuildJob.collectArray externJobs
 
   -- Build dynlib
-  linksJob.bindM fun links => do
-  modDynlibsJob.bindM fun modDynlibs => do
-  externDynlibsJob.mapM fun externDynlibs => do
-    addLeanTrace
-    addPlatformTrace -- shared libraries are platform-dependent artifacts
-    addPureTrace mod.linkArgs
-    buildFileUnlessUpToDate' mod.dynlibFile do
-      let lean ← getLeanInstall
-      let libDirs := pkgLibDirs ++ externDynlibs.filterMap (·.dir?)
-      let libNames := modDynlibs.map (·.name) ++ externDynlibs.map (·.name)
+  linksJob.bindAsync fun links linksTrace => do
+  modDynlibsJob.bindAsync fun modDynlibs modLibsTrace => do
+  externDynlibsJob.bindSync fun externDynlibs externLibsTrace => do
+    let libNames := modDynlibs.map (·.name) ++ externDynlibs.map (·.name)
+    let libDirs := pkgLibDirs ++ externDynlibs.filterMap (·.dir?)
+    let depTrace :=
+      linksTrace.mix <| modLibsTrace.mix <| externLibsTrace.mix
+      <| (← getLeanTrace).mix <| (pureHash mod.linkArgs).mix <|
+      platformTrace
+    let trace ← buildFileUnlessUpToDate mod.dynlibFile depTrace do
       let args :=
         links.map toString ++
         libDirs.map (s!"-L{·}") ++ libNames.map (s!"-l{·}") ++
-        mod.weakLinkArgs ++ mod.linkArgs ++ lean.ccLinkSharedFlags
-      compileSharedLib mod.dynlibFile args lean.cc
-    return ⟨mod.dynlibFile, mod.dynlibName⟩
+        mod.weakLinkArgs ++ mod.linkArgs
+      compileSharedLib mod.dynlibFile args (← getLeanc)
+    return (⟨mod.dynlibFile, mod.dynlibName⟩, trace)
 
 /-- The `ModuleFacetConfig` for the builtin `dynlibFacet`. -/
 def Module.dynlibFacetConfig : ModuleFacetConfig dynlibFacet :=

src/lake/Lake/Build/Package.lean

@@ -34,7 +34,7 @@ def Package.depsFacetConfig : PackageFacetConfig depsFacet :=
 Tries to download and unpack the package's cached build archive
 (e.g., from Reservoir or GitHub).
 -/
-private def Package.fetchOptBuildCacheCore (self : Package) : FetchM (Job Bool) := do
+private def Package.fetchOptBuildCacheCore (self : Package) : FetchM (BuildJob Bool) := do
   if self.preferReleaseBuild then
     self.optGitHubRelease.fetch
   else
@@ -45,7 +45,7 @@ def Package.optBuildCacheFacetConfig : PackageFacetConfig optBuildCacheFacet :=
   mkFacetJobConfig (·.fetchOptBuildCacheCore)
 
 /-- Tries to download the package's build cache (if configured). -/
-def Package.maybeFetchBuildCache (self : Package) : FetchM (Job Bool) := do
+def Package.maybeFetchBuildCache (self : Package) : FetchM (BuildJob Bool) := do
   let shouldFetch :=
     (← getTryCache) &&
     !(← self.buildDir.pathExists) && -- do not automatically clobber prebuilt artifacts
@@ -70,7 +70,7 @@ Tries to download and unpack the package's cached build archive
 -/
 def Package.maybeFetchBuildCacheWithWarning (self : Package) := do
   let job ← self.maybeFetchBuildCache
-  job.mapM fun success => do
+  job.bindSync fun success t => do
     unless success do
       if self.preferReleaseBuild then
         let details ← self.optFacetDetails optGitHubReleaseFacet
@@ -78,6 +78,7 @@ def Package.maybeFetchBuildCacheWithWarning (self : Package) := do
       else
         let details ← self.optFacetDetails optReservoirBarrelFacet
         logVerbose s!"building from source; failed to fetch Reservoir build{details}"
+    return ((), t)
 
 @[deprecated maybeFetchBuildCacheWithWarning (since := "2024-09-27")]
 def Package.fetchOptRelease := @maybeFetchBuildCacheWithWarning
@@ -86,15 +87,15 @@ def Package.fetchOptRelease := @maybeFetchBuildCacheWithWarning
 Build the `extraDepTargets` for the package.
 Also, if the package is a dependency, maybe fetch its build cache.
 -/
-def Package.recBuildExtraDepTargets (self : Package) : FetchM (Job Unit) :=
+def Package.recBuildExtraDepTargets (self : Package) : FetchM (BuildJob Unit) :=
   withRegisterJob s!"{self.name}:extraDep" do
-  let mut job := Job.nil
+  let mut job := BuildJob.nil
   -- Fetch build cache if this package is a dependency
   if self.name ≠ (← getWorkspace).root.name then
-    job := job.add (← self.maybeFetchBuildCacheWithWarning)
+    job := job.add <| ← self.maybeFetchBuildCacheWithWarning
   -- Build this package's extra dep targets
   for target in self.extraDepTargets do
-    job := job.mix (← self.fetchTargetJob target)
+    job := job.mix <| ← self.fetchTargetJob target
   return job
 
 /-- The `PackageFacetConfig` for the builtin `dynlibFacet`. -/
@@ -145,28 +146,29 @@ def Package.fetchBuildArchive
 private def Package.mkOptBuildArchiveFacetConfig
   {facet : Name} (archiveFile : Package → FilePath)
   (getUrl : Package → JobM String) (headers : Array String := #[])
-  [FamilyDef PackageData facet (Job Bool)]
+  [FamilyDef PackageData facet (BuildJob Bool)]
 : PackageFacetConfig facet := mkFacetJobConfig fun pkg =>
   withRegisterJob s!"{pkg.name}:{facet}" (optional := true) <| Job.async do
   try
     let url ← getUrl pkg
     pkg.fetchBuildArchive url (archiveFile pkg) headers
-    return true
+    return (true, .nil)
   catch _ =>
     updateAction .fetch
-    return false
+    return (false, .nil)
 
 @[inline]
 private def Package.mkBuildArchiveFacetConfig
   {facet : Name} (optFacet : Name) (what : String)
-  [FamilyDef PackageData facet (Job Unit)]
-  [FamilyDef PackageData optFacet (Job Bool)]
+  [FamilyDef PackageData facet (BuildJob Unit)]
+  [FamilyDef PackageData optFacet (BuildJob Bool)]
 : PackageFacetConfig facet :=
   mkFacetJobConfig fun pkg =>
     withRegisterJob s!"{pkg.name}:{facet}" do
-      (← fetch <| pkg.facet optFacet).mapM fun success => do
+      (← fetch <| pkg.facet optFacet).bindSync fun success t => do
         unless success do
           error s!"failed to fetch {what}{← pkg.optFacetDetails optFacet}"
+        return ((), t)
 
 /-- The `PackageFacetConfig` for the builtin `buildCacheFacet`. -/
 def Package.buildCacheFacetConfig : PackageFacetConfig buildCacheFacet :=
@@ -198,11 +200,9 @@ abbrev Package.releaseFacetConfig := gitHubReleaseFacetConfig
 Perform a build job after first checking for an (optional) cached build
 for the package (e.g., from Reservoir or GitHub).
 -/
-def Package.afterBuildCacheAsync (self : Package) (build : JobM (Job α)) : FetchM (Job α) := do
+def Package.afterBuildCacheAsync (self : Package) (build : SpawnM (Job α)) : FetchM (Job α) := do
   if self.name ≠ (← getRootPackage).name then
-    (← self.maybeFetchBuildCache).bindM fun _ => do
-      setTrace nilTrace -- ensure both branches start with the same trace
-      build
+    (← self.maybeFetchBuildCache).bindAsync fun _ _ => build
   else
     build
 
@@ -215,9 +215,7 @@ def Package.afterReleaseAsync := @afterBuildCacheAsync
 -/
 def Package.afterBuildCacheSync (self : Package) (build : JobM α) : FetchM (Job α) := do
   if self.name ≠ (← getRootPackage).name then
-    (← self.maybeFetchBuildCache).mapM fun _  => do
-      setTrace nilTrace -- ensure both branches start with the same trace
-      build
+    (← self.maybeFetchBuildCache).bindSync fun _ _ => build
   else
     Job.async build
 

src/lake/Lake/Build/Run.lean

@@ -101,7 +101,7 @@ def renderProgress (running unfinished : Array OpaqueJob) (h : 0 < unfinished.si
 def reportJob (job : OpaqueJob) : MonitorM PUnit := do
   let {jobNo, ..} ← get
   let {totalJobs, failLv, outLv, showOptional, out, useAnsi, showProgress, minAction, ..} ← read
-  let {task, caption, optional} := job
+  let {task, caption, optional} := job.toJob
   let {log, action, ..} := task.get.state
   let maxLv := log.maxLv
   let failed := log.hasEntries ∧ maxLv ≥ failLv
@@ -249,7 +249,7 @@ def Workspace.runFetchM
 
 /-- Run a build function in the Workspace's context and await the result. -/
 @[inline] def Workspace.runBuild
-  (ws : Workspace) (build : FetchM (Job α)) (cfg : BuildConfig := {})
+  (ws : Workspace) (build : FetchM (BuildJob α)) (cfg : BuildConfig := {})
 : IO α := do
   let job ← ws.runFetchM build cfg
   let some a ← job.wait? | error "build failed"
@@ -257,6 +257,6 @@ def Workspace.runFetchM
 
 /-- Produce a build job in the Lake monad's workspace and await the result. -/
 @[inline] def runBuild
-  (build : FetchM (Job α)) (cfg : BuildConfig := {})
+  (build : FetchM (BuildJob α)) (cfg : BuildConfig := {})
 : LakeT IO α := do
   (← getWorkspace).runBuild build cfg

src/lake/Lake/Build/Targets.lean

@@ -17,45 +17,40 @@ open System (FilePath)
 /-! ## Package Facets & Targets -/
 
 /-- Fetch the build job of the specified package target. -/
-def Package.fetchTargetJob
-  (self : Package) (target : Name)
-: FetchM OpaqueJob :=  do
+def Package.fetchTargetJob (self : Package)
+(target : Name) : FetchM (BuildJob Unit) :=  do
   let some config := self.findTargetConfig? target
     | error s!"package '{self.name}' has no target '{target}'"
   return config.getJob (← fetch <| self.target target)
 
 /-- Fetch the build result of a target. -/
-protected def TargetDecl.fetch
-  (self : TargetDecl)
-  [FamilyOut CustomData (self.pkg, self.name) α]
-: FetchM α := do
+protected def TargetDecl.fetch (self : TargetDecl)
+[FamilyOut CustomData (self.pkg, self.name) α] : FetchM α := do
   let some pkg ← findPackage? self.pkg
     | error s!"package '{self.pkg}' of target '{self.name}' does not exist in workspace"
   fetch <| pkg.target self.name
 
 /-- Fetch the build job of the target. -/
-def TargetDecl.fetchJob (self : TargetDecl) : FetchM OpaqueJob :=  do
+def TargetDecl.fetchJob (self : TargetDecl) : FetchM (BuildJob Unit) :=  do
   let some pkg ← findPackage? self.pkg
     | error s!"package '{self.pkg}' of target '{self.name}' does not exist in workspace"
   return self.config.getJob (← fetch <| pkg.target self.name)
 
 /-- Fetch the build result of a package facet. -/
-@[inline] protected def PackageFacetDecl.fetch
-  (pkg : Package) (self : PackageFacetDecl) [FamilyOut PackageData self.name α]
-: FetchM α := fetch <| pkg.facet self.name
+@[inline] protected def PackageFacetDecl.fetch (pkg : Package)
+(self : PackageFacetDecl) [FamilyOut PackageData self.name α] : FetchM α := do
+  fetch <| pkg.facet self.name
 
 /-- Fetch the build job of a package facet. -/
-def PackageFacetConfig.fetchJob
-  (pkg : Package) (self : PackageFacetConfig name)
-: FetchM OpaqueJob :=  do
+def PackageFacetConfig.fetchJob (pkg : Package)
+(self : PackageFacetConfig name) : FetchM (BuildJob Unit) :=  do
   let some getJob := self.getJob?
     | error s!"package facet '{name}' has no associated build job"
   return getJob <| ← fetch <| pkg.facet self.name
 
 /-- Fetch the build job of a library facet. -/
-def Package.fetchFacetJob
-  (name : Name) (self : Package)
-: FetchM OpaqueJob :=  do
+def Package.fetchFacetJob (name : Name)
+(self : Package) : FetchM (BuildJob Unit) :=  do
   let some config := (← getWorkspace).packageFacetConfigs.find? name
     | error s!"package facet '{name}' does not exist in workspace"
   inline <| config.fetchJob self
@@ -63,22 +58,20 @@ def Package.fetchFacetJob
 /-! ## Module Facets -/
 
 /-- Fetch the build result of a module facet. -/
-@[inline] protected def ModuleFacetDecl.fetch
-  (mod : Module) (self : ModuleFacetDecl) [FamilyOut ModuleData self.name α]
-: FetchM α := fetch <| mod.facet self.name
+@[inline] protected def ModuleFacetDecl.fetch (mod : Module)
+(self : ModuleFacetDecl) [FamilyOut ModuleData self.name α] : FetchM α := do
+  fetch <| mod.facet self.name
 
 /-- Fetch the build job of a module facet. -/
-def ModuleFacetConfig.fetchJob
-  (mod : Module) (self : ModuleFacetConfig name)
-: FetchM OpaqueJob :=  do
+def ModuleFacetConfig.fetchJob (mod : Module)
+(self : ModuleFacetConfig name) : FetchM (BuildJob Unit) :=  do
   let some getJob := self.getJob?
     | error s!"module facet '{self.name}' has no associated build job"
   return getJob <| ← fetch <| mod.facet self.name
 
 /-- Fetch the build job of a module facet. -/
 def Module.fetchFacetJob
-  (name : Name) (self : Module)
-: FetchM OpaqueJob :=  do
+(name : Name) (self : Module) : FetchM (BuildJob Unit) :=  do
   let some config := (← getWorkspace).moduleFacetConfigs.find? name
     | error s!"library facet '{name}' does not exist in workspace"
   inline <| config.fetchJob self
@@ -93,22 +86,20 @@ def Module.fetchFacetJob
   return lib
 
 /-- Fetch the build result of a library facet. -/
-@[inline] protected def LibraryFacetDecl.fetch
-  (lib : LeanLib) (self : LibraryFacetDecl) [FamilyOut LibraryData self.name α]
-: FetchM α := fetch <| lib.facet self.name
+@[inline] protected def LibraryFacetDecl.fetch (lib : LeanLib)
+(self : LibraryFacetDecl) [FamilyOut LibraryData self.name α] : FetchM α := do
+  fetch <| lib.facet self.name
 
 /-- Fetch the build job of a library facet. -/
-def LibraryFacetConfig.fetchJob
-  (lib : LeanLib) (self : LibraryFacetConfig name)
-: FetchM OpaqueJob :=  do
+def LibraryFacetConfig.fetchJob (lib : LeanLib)
+(self : LibraryFacetConfig name) : FetchM (BuildJob Unit) :=  do
   let some getJob := self.getJob?
     | error s!"library facet '{self.name}' has no associated build job"
   return getJob <| ← fetch <| lib.facet self.name
 
 /-- Fetch the build job of a library facet. -/
-def LeanLib.fetchFacetJob
-  (name : Name) (self : LeanLib)
-: FetchM OpaqueJob :=  do
+def LeanLib.fetchFacetJob (name : Name)
+(self : LeanLib) : FetchM (BuildJob Unit) :=  do
   let some config := (← getWorkspace).libraryFacetConfigs.find? name
     | error s!"library facet '{name}' does not exist in workspace"
   inline <| config.fetchJob self
@@ -116,17 +107,17 @@ def LeanLib.fetchFacetJob
 /-! ## Lean Executable Target -/
 
 /-- Get the Lean executable in the workspace with the configuration's name. -/
-@[inline] def LeanExeConfig.get
-  (self : LeanExeConfig) [Monad m] [MonadError m] [MonadLake m]
-: m LeanExe := do
+@[inline] def LeanExeConfig.get (self : LeanExeConfig)
+[Monad m] [MonadError m] [MonadLake m] : m LeanExe := do
   let some exe ← findLeanExe? self.name
     | error s!"Lean executable '{self.name}' does not exist in the workspace"
   return exe
 
 /-- Fetch the build of the Lean executable. -/
-@[inline] def LeanExeConfig.fetch (self : LeanExeConfig) : FetchM (Job FilePath) := do
+@[inline] def LeanExeConfig.fetch
+(self : LeanExeConfig) : FetchM (BuildJob FilePath) := do
   (← self.get).exe.fetch
 
 /-- Fetch the build of the Lean executable. -/
-@[inline] def LeanExe.fetch (self : LeanExe) : FetchM (Job FilePath) :=
+@[inline] def LeanExe.fetch (self : LeanExe) : FetchM (BuildJob FilePath) :=
   self.exe.fetch

src/lake/Lake/CLI/Build.lean

@@ -4,7 +4,7 @@ Released under Apache 2.0 license as described in the file LICENSE.
 Authors: Mac Malone
 -/
 prelude
-import Lake.Config.Monad
+import Lake.Build.Index
 import Lake.CLI.Error
 
 namespace Lake
@@ -14,32 +14,32 @@ open Lean (Name)
 
 structure BuildSpec where
   info : BuildInfo
-  getBuildJob : BuildData info.key → OpaqueJob
+  getBuildJob : BuildData info.key → BuildJob Unit
 
-@[inline] def BuildData.toJob
-  [FamilyOut BuildData k (Job α)] (data : BuildData k)
-: OpaqueJob :=
-  ofFamily data |>.toOpaque
+@[inline] def BuildData.toBuildJob
+  [FamilyOut BuildData k (BuildJob α)] (data : BuildData k)
+: BuildJob Unit :=
+  discard <| ofFamily data
 
 @[inline] def mkBuildSpec
-  (info : BuildInfo) [FamilyOut BuildData info.key (Job α)]
+  (info : BuildInfo) [FamilyOut BuildData info.key (BuildJob α)]
 : BuildSpec :=
-  {info, getBuildJob := BuildData.toJob}
+  {info, getBuildJob := BuildData.toBuildJob}
 
 @[inline] def mkConfigBuildSpec
   (facetType : String) (info : BuildInfo)
   (config : FacetConfig Fam ι facet) (h : BuildData info.key = Fam facet)
 : Except CliError BuildSpec := do
-  let some getBuildJob := config.getJob?
+  let some getJob := config.getJob?
     | throw <| CliError.nonCliFacet facetType facet
-  return {info, getBuildJob := h ▸ getBuildJob}
+  return {info, getBuildJob := h ▸ getJob}
 
-@[inline] protected def BuildSpec.fetch (self : BuildSpec) : FetchM OpaqueJob := do
+@[inline] protected def BuildSpec.fetch (self : BuildSpec) : FetchM (BuildJob Unit) := do
   maybeRegisterJob (self.info.key.toSimpleString) <| ← do
     self.getBuildJob <$> self.info.fetch
 
-def buildSpecs (specs : Array BuildSpec) : FetchM (Job Unit) := do
-  return .mixArray (← specs.mapM (·.fetch))
+def buildSpecs (specs : Array BuildSpec) : FetchM (BuildJob Unit) := do
+  BuildJob.mixArray (← specs.mapM (·.fetch))
 
 /-! ## Parsing CLI Build Target Specifiers -/
 

src/lake/Lake/CLI/Help.lean

@@ -46,7 +46,6 @@ BASIC OPTIONS:
   --old                 only rebuild modified modules (ignore transitive deps)
   --rehash, -H          hash all files for traces (do not trust `.hash` files)
   --update, -U          update dependencies on load (e.g., before a build)
-  --packages=file       JSON file of package entries that override the manifest
   --reconfigure, -R     elaborate configuration files instead of using OLeans
   --keep-toolchain      do not update toolchain on workspace update
   --no-build            exit immediately if a build target is not up-to-date

src/lake/Lake/CLI/Main.lean

@@ -34,7 +34,6 @@ structure LakeOptions where
   leanInstall? : Option LeanInstall := none
   lakeInstall? : Option LakeInstall := none
   configOpts : NameMap String := {}
-  packageOverrides : Array PackageEntry := #[]
   subArgs : List String := []
   wantsHelp : Bool := false
   verbosity : Verbosity := .normal
@@ -80,7 +79,6 @@ def LakeOptions.mkLoadConfig (opts : LakeOptions) : EIO CliError LoadConfig :=
     lakeEnv := ← opts.computeEnv
     wsDir := opts.rootDir
     relConfigFile := opts.configFile
-    packageOverrides := opts.packageOverrides
     lakeOpts := opts.configOpts
     leanOpts := Lean.Options.empty
     reconfigure := opts.reconfigure
@@ -198,11 +196,6 @@ def lakeLongOption : (opt : String) → CliM PUnit
   modifyThe LakeOptions ({· with failLv})
 | "--ansi"        => modifyThe LakeOptions ({· with ansiMode := .ansi})
 | "--no-ansi"     => modifyThe LakeOptions ({· with ansiMode := .noAnsi})
-| "--packages"    => do
-  let file ← takeOptArg "--packages" "package overrides file"
-  let overrides ← Manifest.loadEntries file
-  modifyThe LakeOptions fun opts =>
-    {opts with packageOverrides := opts.packageOverrides ++ overrides}
 | "--dir"         => do
   let rootDir ← takeOptArg "--dir" "path"
   modifyThe LakeOptions ({· with rootDir})

src/lake/Lake/Config/ExternLibConfig.lean

@@ -13,7 +13,7 @@ open Lean System
 /-- A external library's declarative configuration. -/
 structure ExternLibConfig (pkgName name : Name) where
   /-- The library's build data. -/
-  getJob : CustomData (pkgName, .str name "static") → Job FilePath
+  getJob : CustomData (pkgName, .str name "static") → BuildJob FilePath
   deriving Inhabited
 
 /-- A dependently typed configuration based on its registered package and name. -/

src/lake/Lake/Config/FacetConfig.lean

@@ -14,7 +14,7 @@ structure FacetConfig (DataFam : Name → Type) (ι : Type) (name : Name) : Type
   /-- The facet's build (function). -/
   build : ι → FetchM (DataFam name)
   /-- Does this facet produce an associated asynchronous job? -/
-  getJob? : Option (DataFam name → OpaqueJob)
+  getJob? : Option (DataFam name → BuildJob Unit)
   deriving Inhabited
 
 protected abbrev FacetConfig.name (_ : FacetConfig DataFam ι name) := name
@@ -26,10 +26,10 @@ protected abbrev FacetConfig.name (_ : FacetConfig DataFam ι name) := name
   getJob? := none
 
 /-- A smart constructor for facet configurations that generate jobs for the CLI. -/
-@[inline] def mkFacetJobConfig (build : ι → FetchM (Job α))
-[h : FamilyOut Fam facet (Job α)] : FacetConfig Fam ι facet where
+@[inline] def mkFacetJobConfig (build : ι → FetchM (BuildJob α))
+[h : FamilyOut Fam facet (BuildJob α)] : FacetConfig Fam ι facet where
   build := cast (by rw [← h.family_key_eq_type]) build
-  getJob? := some fun data => ofFamily data |>.toOpaque
+  getJob? := some fun data => discard <| ofFamily data
 
 /-- A dependently typed configuration based on its registered name. -/
 structure NamedConfigDecl (β : Name → Type u) where

src/lake/Lake/Config/LeanExeConfig.lean

@@ -71,7 +71,7 @@ structure LeanExeConfig extends LeanConfig where
   `Module.oFacet`. That is, the  object file compiled from the Lean source,
   potentially with exported Lean symbols.
   -/
-  nativeFacets (shouldExport : Bool) : Array (ModuleFacet (Job FilePath)) :=
+  nativeFacets (shouldExport : Bool) : Array (ModuleFacet (BuildJob FilePath)) :=
     #[if shouldExport then Module.oExportFacet else Module.oFacet]
 
 deriving Inhabited

src/lake/Lake/Config/LeanLib.lean

@@ -97,7 +97,7 @@ Otherwise, falls back to the package's.
   self.config.defaultFacets
 
 /-- The library's `nativeFacets` configuration. -/
-@[inline] def nativeFacets (self : LeanLib) (shouldExport : Bool) : Array (ModuleFacet (Job FilePath)) :=
+@[inline] def nativeFacets (self : LeanLib) (shouldExport : Bool) : Array (ModuleFacet (BuildJob FilePath)) :=
   self.config.nativeFacets shouldExport
 
 /--

src/lake/Lake/Config/LeanLibConfig.lean

@@ -79,7 +79,7 @@ structure LeanLibConfig extends LeanConfig where
   `Module.oFacet`. That is, the  object files compiled from the Lean sources,
   potentially with exported Lean symbols.
   -/
-  nativeFacets (shouldExport : Bool) : Array (ModuleFacet (Job FilePath)) :=
+  nativeFacets (shouldExport : Bool) : Array (ModuleFacet (BuildJob FilePath)) :=
     #[if shouldExport then Module.oExportFacet else Module.oFacet]
 
 deriving Inhabited

src/lake/Lake/Config/Module.lean

@@ -148,7 +148,7 @@ def dynlibSuffix := "-1"
 @[inline] def shouldPrecompile (self : Module) : Bool :=
   self.lib.precompileModules
 
-@[inline] def nativeFacets (self : Module) (shouldExport : Bool) : Array (ModuleFacet (Job FilePath)) :=
+@[inline] def nativeFacets (self : Module) (shouldExport : Bool) : Array (ModuleFacet (BuildJob FilePath)) :=
   self.lib.nativeFacets shouldExport
 
 /-! ## Trace Helpers -/

src/lake/Lake/Config/Opaque.lean

@@ -5,7 +5,7 @@ Authors: Mac Malone
 -/
 prelude
 import Lake.Util.Name
-import Lake.Util.OpaqueType
+import Lake.Util.Opaque
 
 namespace Lake
 

src/lake/Lake/Config/TargetConfig.lean

@@ -14,15 +14,15 @@ structure TargetConfig (pkgName name : Name) : Type where
   /-- The target's build function. -/
   build : (pkg : NPackage pkgName) → FetchM (CustomData (pkgName, name))
   /-- The target's resulting build job. -/
-  getJob : CustomData (pkgName, name) → OpaqueJob
+  getJob : CustomData (pkgName, name) → BuildJob Unit
   deriving Inhabited
 
 /-- A smart constructor for target configurations that generate CLI targets. -/
 @[inline] def mkTargetJobConfig
-(build : (pkg : NPackage pkgName) → FetchM (Job α))
-[h : FamilyOut CustomData (pkgName, name) (Job α)] : TargetConfig pkgName name where
+(build : (pkg : NPackage pkgName) → FetchM (BuildJob α))
+[h : FamilyOut CustomData (pkgName, name) (BuildJob α)] : TargetConfig pkgName name where
   build := cast (by rw [← h.family_key_eq_type]) build
-  getJob := fun data => ofFamily data |>.toOpaque
+  getJob := fun data => discard <| ofFamily data
 
 /-- A dependently typed configuration based on its registered package and name. -/
 structure TargetDecl where

src/lake/Lake/Config/Workspace.lean

@@ -74,10 +74,6 @@ namespace Workspace
 @[inline] def manifestFile (self : Workspace) : FilePath :=
   self.root.manifestFile
 
-/-- The path to the workspace file used to configure automatic package overloads. -/
-@[inline] def packageOverridesFile (self : Workspace) : FilePath :=
-  self.lakeDir / "package-overrides.json"
-
 /-- Add a package to the workspace. -/
 def addPackage (pkg : Package) (self : Workspace) : Workspace :=
   {self with packages := self.packages.push pkg, packageMap := self.packageMap.insert pkg.name pkg}

src/lake/Lake/DSL/Targets.lean

@@ -24,8 +24,8 @@ syntax buildDeclSig :=
 
 abbrev mkModuleFacetDecl
   (α) (facet : Name)
-  [FamilyDef ModuleData facet (Job α)]
-  (f : Module → FetchM (Job α))
+  [FamilyDef ModuleData facet (BuildJob α)]
+  (f : Module → FetchM (BuildJob α))
 : ModuleFacetDecl := .mk facet <| mkFacetJobConfig fun mod => do
   withRegisterJob (mod.facet facet |>.key.toSimpleString)
     (f mod)
@@ -35,7 +35,7 @@ Define a new module facet. Has one form:
 
 ```lean
 module_facet «facet-name» (mod : Module) : α :=
-  /- build term of type `FetchM (Job α)` -/
+  /- build term of type `FetchM (BuildJob α)` -/
 ```
 
 The `mod` parameter (and its type specifier) is optional.
@@ -51,7 +51,7 @@ kw:"module_facet " sig:buildDeclSig : command => withRef kw do
     let facet := Name.quoteFrom id id.getId
     let declId := mkIdentFrom id <| id.getId.modifyBase (.str · "_modFacet")
     let mod ← expandOptSimpleBinder mod?
-    `(module_data $id : Job $ty
+    `(module_data $id : BuildJob $ty
       $[$doc?:docComment]? @[$attrs,*] abbrev $declId :=
         Lake.DSL.mkModuleFacetDecl $ty $facet (fun $mod => $defn)
       $[$wds?:whereDecls]?)
@@ -59,8 +59,8 @@ kw:"module_facet " sig:buildDeclSig : command => withRef kw do
 
 abbrev mkPackageFacetDecl
   (α) (facet : Name)
-  [FamilyDef PackageData facet (Job α)]
-  (f : Package → FetchM (Job α))
+  [FamilyDef PackageData facet (BuildJob α)]
+  (f : Package → FetchM (BuildJob α))
 : PackageFacetDecl := .mk facet <| mkFacetJobConfig fun pkg => do
   withRegisterJob (pkg.facet facet |>.key.toSimpleString)
     (f pkg)
@@ -70,7 +70,7 @@ Define a new package facet. Has one form:
 
 ```lean
 package_facet «facet-name» (pkg : Package) : α :=
-  /- build term of type `FetchM (Job α)` -/
+  /- build term of type `FetchM (BuildJob α)` -/
 ```
 
 The `pkg` parameter (and its type specifier) is optional.
@@ -86,7 +86,7 @@ kw:"package_facet " sig:buildDeclSig : command => withRef kw do
     let facet := Name.quoteFrom id id.getId
     let declId := mkIdentFrom id <| id.getId.modifyBase (.str · "_pkgFacet")
     let pkg ← expandOptSimpleBinder pkg?
-    `(package_data $id : Job $ty
+    `(package_data $id : BuildJob $ty
       $[$doc?]? @[$attrs,*] abbrev $declId :=
         Lake.DSL.mkPackageFacetDecl $ty $facet (fun $pkg => $defn)
       $[$wds?:whereDecls]?)
@@ -94,8 +94,8 @@ kw:"package_facet " sig:buildDeclSig : command => withRef kw do
 
 abbrev mkLibraryFacetDecl
   (α) (facet : Name)
-  [FamilyDef LibraryData facet (Job α)]
-  (f : LeanLib → FetchM (Job α))
+  [FamilyDef LibraryData facet (BuildJob α)]
+  (f : LeanLib → FetchM (BuildJob α))
 : LibraryFacetDecl := .mk facet <| mkFacetJobConfig fun lib => do
   withRegisterJob (lib.facet facet |>.key.toSimpleString)
     (f lib)
@@ -105,7 +105,7 @@ Define a new library facet. Has one form:
 
 ```lean
 library_facet «facet-name» (lib : LeanLib) : α :=
-  /- build term of type `FetchM (Job α)` -/
+  /- build term of type `FetchM (BuildJob α)` -/
 ```
 
 The `lib` parameter (and its type specifier) is optional.
@@ -121,7 +121,7 @@ kw:"library_facet " sig:buildDeclSig : command => withRef kw do
     let facet := Name.quoteFrom id id.getId
     let declId := mkIdentFrom id <| id.getId.modifyBase (.str · "_libFacet")
     let lib ← expandOptSimpleBinder lib?
-    `(library_data $id : Job $ty
+    `(library_data $id : BuildJob $ty
       $[$doc?]? @[$attrs,*] abbrev $declId : LibraryFacetDecl :=
         Lake.DSL.mkLibraryFacetDecl $ty $facet (fun $lib => $defn)
       $[$wds?:whereDecls]?)
@@ -133,8 +133,8 @@ kw:"library_facet " sig:buildDeclSig : command => withRef kw do
 
 abbrev mkTargetDecl
   (α) (pkgName target : Name)
-  [FamilyDef CustomData (pkgName, target) (Job α)]
-  (f : NPackage pkgName → FetchM (Job α))
+  [FamilyDef CustomData (pkgName, target) (BuildJob α)]
+  (f : NPackage pkgName → FetchM (BuildJob α))
 : TargetDecl := .mk pkgName target <| mkTargetJobConfig fun pkg => do
   withRegisterJob (pkg.target target |>.key.toSimpleString)
     (f pkg)
@@ -144,7 +144,7 @@ Define a new custom target for the package. Has one form:
 
 ```lean
 target «target-name» (pkg : NPackage _package.name) : α :=
-  /- build term of type `FetchM (Job α)` -/
+  /- build term of type `FetchM (BuildJob α)` -/
 ```
 
 The `pkg` parameter (and its type specifier) is optional.
@@ -161,7 +161,7 @@ kw:"target " sig:buildDeclSig : command => do
     let name := Name.quoteFrom id id.getId
     let pkgName := mkIdentFrom id `_package.name
     let pkg ← expandOptSimpleBinder pkg?
-    `(family_def $id : CustomData ($pkgName, $name) := Job $ty
+    `(family_def $id : CustomData ($pkgName, $name) := BuildJob $ty
       $[$doc?]? @[$attrs,*] abbrev $id :=
         Lake.DSL.mkTargetDecl $ty $pkgName $name (fun $pkg => $defn)
       $[$wds?:whereDecls]?)
@@ -230,7 +230,7 @@ Define a new external library target for the package. Has one form:
 
 ```lean
 extern_lib «target-name» (pkg : NPackage _package.name) :=
-  /- build term of type `FetchM (Job FilePath)` -/
+  /- build term of type `FetchM (BuildJob FilePath)` -/
 ```
 
 The `pkg` parameter (and its type specifier) is optional.

src/lake/Lake/Load/Config.lean

@@ -6,8 +6,10 @@ Authors: Mac Malone
 prelude
 import Lean.Data.Name
 import Lean.Data.Options
+import Lake.Config.Defaults
 import Lake.Config.Env
-import Lake.Load.Manifest
+import Lake.Util.Log
+import Lake.Util.Version
 
 namespace Lake
 open System Lean
@@ -28,8 +30,6 @@ structure LoadConfig where
   relPkgDir : FilePath := "."
   /-- The package's Lake configuration file (relative to the package directory). -/
   relConfigFile : FilePath := defaultConfigFile
-  /-- Additional package overrides for this workspace load. -/
-  packageOverrides : Array PackageEntry := #[]
   /-- A set of key-value Lake configuration options (i.e., `-K` settings). -/
   lakeOpts : NameMap String := {}
   /-- The Lean options with which to elaborate the configuration file. -/

src/lake/Lake/Load/Manifest.lean

@@ -9,7 +9,8 @@ import Lake.Util.Name
 import Lake.Util.FilePath
 import Lake.Util.JsonObject
 import Lake.Util.Version
-import Lake.Config.Defaults
+import Lake.Load.Config
+import Lake.Config.Workspace
 
 open System Lean
 
@@ -190,44 +191,37 @@ protected def toJson (self : Manifest) : Json :=
 
 instance : ToJson Manifest := ⟨Manifest.toJson⟩
 
-def getVersion (obj : JsonObject) : Except String SemVerCore := do
-  let ver : Json ← obj.get "version" <|> obj.get "schemaVersion"
+protected def fromJson? (json : Json) : Except String Manifest := do
+  let obj ← JsonObject.fromJson? json
   let ver : SemVerCore ←
-    match ver with
+    match (← obj.get "version" : Json) with
     | (n : Nat) => pure {minor := n}
     | (s : String) => StdVer.parse s
-    | ver => throw s!"invalid version '{ver}'; \
+    | ver => throw s!"unknown manifest version format '{ver}'; \
       you may need to update your 'lean-toolchain'"
   if ver.major > 1 then
-    throw s!"schema version '{ver}' is of a higher major version than this \
+    throw s!"manifest version '{ver}' is of a higher major version than this \
       Lake's '{Manifest.version}'; you may need to update your 'lean-toolchain'"
   else if ver < {minor := 5} then
     throw s!"incompatible manifest version '{ver}'"
   else
-    return ver
-
-def getPackages (ver : StdVer) (obj : JsonObject) : Except String (Array PackageEntry) := do
-  if ver < {minor := 7} then
-    (·.map PackageEntry.ofV6) <$> obj.getD "packages" #[]
-  else
-    obj.getD "packages" #[]
-
-protected def fromJson? (json : Json) : Except String Manifest := do
-  let obj ← JsonObject.fromJson? json
-  let ver ← getVersion obj
-  let name ← obj.getD "name" Name.anonymous
-  let lakeDir ← obj.getD "lakeDir" defaultLakeDir
-  let packagesDir? ← obj.get? "packagesDir"
-  let packages ← getPackages ver obj
-  return {name, lakeDir, packagesDir?, packages}
+    let name ← obj.getD "name" Name.anonymous
+    let lakeDir ← obj.getD "lakeDir" defaultLakeDir
+    let packagesDir? ← obj.get? "packagesDir"
+    let packages ←
+      if ver < {minor := 7} then
+        (·.map PackageEntry.ofV6) <$> obj.getD "packages" #[]
+      else
+        obj.getD "packages" #[]
+    return {name, lakeDir, packagesDir?, packages}
 
 instance : FromJson Manifest := ⟨Manifest.fromJson?⟩
 
 /-- Parse a `Manifest` from a string. -/
-def parse (data : String) : Except String Manifest := do
-  match Json.parse data with
+def parse (s : String) : Except String Manifest := do
+  match Json.parse s with
   | .ok json => fromJson? json
-  | .error e => throw s!"invalid JSON: {e}"
+  | .error e => throw s!"manifest is not valid JSON: {e}"
 
 /-- Parse a manifest file. -/
 def load (file : FilePath) : IO Manifest := do
@@ -246,45 +240,7 @@ def load? (file : FilePath) : IO (Option Manifest) := do
   | .error (.noFileOrDirectory ..) => return none
   | .error e => throw e
 
-/-- Serialize the manifest to a JSON file. -/
-def save (self : Manifest) (manifestFile : FilePath) : IO PUnit := do
-  let contents := Json.pretty self.toJson
-  IO.FS.writeFile manifestFile <| contents.push '\n'
-
-@[deprecated save (since := "2024-12-17")] abbrev saveToFile := @save
-
-/-- Deserialize package entries from a (partial) JSON manifest. -/
-def decodeEntries (data : Json)  : Except String (Array PackageEntry) := do
-  let obj ← JsonObject.fromJson? data
-  getPackages (← getVersion obj) obj
-
-/-- Deserialize manifest package entries from a JSON string. -/
-def parseEntries (data : String) : Except String (Array PackageEntry) := do
-  match Json.parse data with
-  | .ok json => decodeEntries json
-  | .error e => throw s!"invalid JSON: {e}"
-
-/-- Deserialize manifest package entries from a JSON file. -/
-def loadEntries (file : FilePath) : IO (Array PackageEntry) := do
-  let contents ← IO.FS.readFile file
-  match inline <| parseEntries contents with
-  | .ok a => return a
-  | .error e => error s!"{file}: {e}"
-
-/--
-Deserialize manifest package entries from a JSON file.
-Returns an empty array if the file does not exist.
--/
-def tryLoadEntries (file : FilePath) : IO (Array PackageEntry) := do
-  match  (← inline (loadEntries file) |>.toBaseIO) with
-  | .ok a => return a
-  | .error (.noFileOrDirectory ..) => return #[]
-  | .error e => error s!"{file}: {e}"
-
-/-- Serialize manifest package entries to a JSON file. -/
-def saveEntries (file : FilePath) (entries : Array PackageEntry)  : IO PUnit := do
-  let contents := Json.pretty <| Json.mkObj [
-    ("schemaVersion", toJson version),
-    ("packages", toJson entries)
-  ]
-  IO.FS.writeFile file <| contents.push '\n'
+/-- Save the manifest as JSON to a file. -/
+def saveToFile (self : Manifest) (manifestFile : FilePath) : IO PUnit := do
+  let jsonString := Json.pretty self.toJson
+  IO.FS.writeFile manifestFile <| jsonString.push '\n'

src/lake/Lake/Load/Resolve.lean

@@ -373,7 +373,7 @@ def Workspace.writeManifest
     packagesDir? := ws.relPkgsDir
     packages := manifestEntries
   }
-  manifest.save ws.manifestFile
+  manifest.saveToFile ws.manifestFile
 
 /-- Run a package's `post_update` hooks. -/
 def Package.runPostUpdateHooks (pkg : Package) : LakeT LoggerIO PUnit := do
@@ -399,12 +399,15 @@ def Workspace.updateAndMaterialize
   return ws
 
 /--
-Check whether entries in the manifest are up-to-date,
+Check whether the manifest exists and
+whether entries in the manifest are up-to-date,
 reporting warnings and/or errors as appropriate.
 -/
 def validateManifest
   (pkgEntries : NameMap PackageEntry) (deps : Array Dependency)
 : LoggerIO PUnit := do
+  if pkgEntries.isEmpty && !deps.isEmpty then
+    error "missing manifest; use `lake update` to generate one"
   deps.forM fun dep => do
     let warnOutOfDate (what : String) :=
       logWarning <|
@@ -426,27 +429,16 @@ downloading and/or updating them as necessary.
 def Workspace.materializeDeps
   (ws : Workspace) (manifest : Manifest)
   (leanOpts : Options := {}) (reconfigure := false)
-  (overrides : Array PackageEntry := #[])
 : LoggerIO Workspace := do
-  -- Load locked dependencies
   if !manifest.packages.isEmpty && manifest.packagesDir? != some (mkRelPathString ws.relPkgsDir) then
     logWarning <|
       "manifest out of date: packages directory changed; \
       use `lake update` to rebuild the manifest \
       (warning: this will update ALL workspace dependencies)"
   let relPkgsDir := manifest.packagesDir?.getD ws.relPkgsDir
-  let mut pkgEntries := mkNameMap PackageEntry
-  pkgEntries := manifest.packages.foldl (init := pkgEntries) fun map entry =>
-    map.insert entry.name entry
+  let pkgEntries : NameMap PackageEntry := manifest.packages.foldl (init := {})
+    fun map entry => map.insert entry.name entry
   validateManifest pkgEntries ws.root.depConfigs
-  let wsOverrides ← Manifest.tryLoadEntries ws.packageOverridesFile
-  pkgEntries := wsOverrides.foldl (init := pkgEntries) fun map entry =>
-    map.insert entry.name entry
-  pkgEntries := overrides.foldl (init := pkgEntries) fun map entry =>
-    map.insert entry.name entry
-  if pkgEntries.isEmpty && !ws.root.depConfigs.isEmpty then
-    error "missing manifest; use `lake update` to generate one"
-  -- Materialize all dependencies
   let ws := ws.addPackage ws.root
   ws.resolveDepsCore fun pkg dep => do
     let ws ← getWorkspace

src/lake/Lake/Load/Workspace.lean

@@ -44,12 +44,12 @@ elaborating its configuration file and resolving its dependencies.
 If `updateDeps` is true, updates the manifest before resolving dependencies.
 -/
 def loadWorkspace (config : LoadConfig) : LoggerIO Workspace := do
-  let {reconfigure, leanOpts, updateDeps, updateToolchain, packageOverrides, ..} := config
+  let {reconfigure, leanOpts, updateDeps, updateToolchain, ..} := config
   let ws ← loadWorkspaceRoot config
   if updateDeps then
     ws.updateAndMaterialize {} leanOpts updateToolchain
   else if let some manifest ← Manifest.load? ws.manifestFile then
-    ws.materializeDeps manifest leanOpts reconfigure packageOverrides
+    ws.materializeDeps manifest leanOpts reconfigure
   else
     ws.updateAndMaterialize {} leanOpts updateToolchain