Generate public API

Update setup and pip_build version

Update build scripts

Update build scripts

.github/workflows/actions.yml

@@ -24,13 +24,13 @@ jobs:
       KERAS_HOME: .github/workflows/config/${{ matrix.backend }}
     steps:
       - uses: actions/checkout@v4
-      - name: Check for changes in keras/applications
+      - name: Check for changes in keras/src/applications
         uses: dorny/paths-filter@v3
         id: filter
         with:
           filters: |
             applications:
-              - 'keras/applications/**'
+              - 'keras/src/applications/**'
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
@@ -54,8 +54,8 @@ jobs:
       - name: Test applications with pytest
         if: ${{ steps.filter.outputs.applications == 'true' }}
         run: |
-          pytest keras/applications --cov=keras/applications
-          coverage xml --include='keras/applications/*' -o apps-coverage.xml
+          pytest keras/src/applications --cov=keras/src/applications
+          coverage xml --include='keras/src/applications/*' -o apps-coverage.xml
       - name: Codecov keras.applications
         if: ${{ steps.filter.outputs.applications == 'true' }}
         uses: codecov/codecov-action@v4
@@ -80,8 +80,8 @@ jobs:
           pytest integration_tests/torch_workflow_test.py
       - name: Test with pytest
         run: |
-          pytest keras --ignore keras/applications --cov=keras
-          coverage xml --omit='keras/applications/*' -o core-coverage.xml
+          pytest keras --ignore keras/src/applications --cov=keras
+          coverage xml --omit='keras/src/applications/*' -o core-coverage.xml
       - name: Codecov keras
         uses: codecov/codecov-action@v4
         with:

.github/workflows/nightly.yml

@@ -55,7 +55,7 @@ jobs:
           pytest integration_tests/torch_workflow_test.py
       - name: Test with pytest
         run: |
-          pytest keras --ignore keras/applications --cov=keras
+          pytest keras --ignore keras/src/applications --cov=keras
 
   format:
     name: Check the code format

.kokoro/github/ubuntu/gpu/build.sh

@@ -34,8 +34,8 @@ then
    python3 -c 'import tensorflow as tf;assert len(tf.config.list_physical_devices("GPU")) > 0'
 
    # TODO: keras/layers/merging/merging_test.py::MergingLayersTest::test_sparse_dot_2d Fatal Python error: Aborted
-   pytest keras --ignore keras/applications \
-               --ignore keras/layers/merging/merging_test.py \
+   pytest keras --ignore keras/src/applications \
+               --ignore keras/src/layers/merging/merging_test.py \
                --cov=keras
 fi
 
@@ -51,11 +51,11 @@ then
    # TODO: keras/layers/merging/merging_test.py::MergingLayersTest::test_sparse_dot_2d Fatal Python error: Aborted
    # TODO: keras/trainers/data_adapters/py_dataset_adapter_test.py::PyDatasetAdapterTest::test_basic_flow0 Fatal Python error: Aborted
    # keras/backend/jax/distribution_lib_test.py is configured for CPU test for now.
-   pytest keras --ignore keras/applications \
-               --ignore keras/layers/merging/merging_test.py \
-               --ignore keras/trainers/data_adapters/py_dataset_adapter_test.py \
-               --ignore keras/backend/jax/distribution_lib_test.py \
-               --ignore keras/distribution/distribution_lib_test.py \
+   pytest keras --ignore keras/src/applications \
+               --ignore keras/src/layers/merging/merging_test.py \
+               --ignore keras/src/trainers/data_adapters/py_dataset_adapter_test.py \
+               --ignore keras/src/backend/jax/distribution_lib_test.py \
+               --ignore keras/src/distribution/distribution_lib_test.py \
                --cov=keras
 fi
 
@@ -68,6 +68,6 @@ then
    # Raise error if GPU is not detected.
    python3 -c 'import torch;assert torch.cuda.is_available()'
 
-   pytest keras --ignore keras/applications \
+   pytest keras --ignore keras/src/applications \
                --cov=keras
 fi

api_gen.py

@@ -0,0 +1,175 @@
+"""Script to create (and optionally install) a `.whl` archive for Keras 3.
+
+Usage:
+
+1. Create a `.whl` file in `dist/`:
+
+```
+python3 pip_build.py
+```
+
+2. Also install the new package immediately after:
+
+```
+python3 pip_build.py --install
+```
+"""
+
+import os
+import shutil
+
+import namex_gen as namex
+
+package = "keras"
+
+
+def ignore_files(_, filenames):
+    return [f for f in filenames if f.endswith("_test.py")]
+
+
+def create_legacy_directory():
+    API_DIR = os.path.join(package, "api")
+    # Make keras/_tf_keras/ by copying keras/
+    tf_keras_dirpath_parent = os.path.join(API_DIR, "_tf_keras")
+    tf_keras_dirpath = os.path.join(tf_keras_dirpath_parent, "keras")
+    os.makedirs(tf_keras_dirpath, exist_ok=True)
+    with open(os.path.join(tf_keras_dirpath_parent, "__init__.py"), "w") as f:
+        f.write("from keras.api._tf_keras import keras\n")
+    with open(os.path.join(API_DIR, "__init__.py")) as f:
+        init_file = f.read()
+        init_file = init_file.replace(
+            "from keras.api import _legacy",
+            "from keras.api import _tf_keras",
+        )
+    with open(os.path.join(API_DIR, "__init__.py"), "w") as f:
+        f.write(init_file)
+    with open(os.path.join(tf_keras_dirpath, "__init__.py"), "w") as f:
+        f.write(init_file)
+    for dirname in os.listdir(API_DIR):
+        dirpath = os.path.join(API_DIR, dirname)
+        if os.path.isdir(dirpath) and dirname not in (
+            "_legacy",
+            "_tf_keras",
+            "src",
+        ):
+            destpath = os.path.join(tf_keras_dirpath, dirname)
+            if os.path.exists(destpath):
+                shutil.rmtree(destpath)
+            shutil.copytree(
+                dirpath,
+                destpath,
+                ignore=ignore_files,
+            )
+
+    # Copy keras/_legacy/ file contents to keras/_tf_keras/keras
+    legacy_submodules = [
+        path[:-3]
+        for path in os.listdir(os.path.join(package, "src", "legacy"))
+        if path.endswith(".py")
+    ]
+    legacy_submodules += [
+        path
+        for path in os.listdir(os.path.join(package, "src", "legacy"))
+        if os.path.isdir(os.path.join(package, "src", "legacy", path))
+    ]
+
+    for root, _, fnames in os.walk(os.path.join(package, "_legacy")):
+        for fname in fnames:
+            if fname.endswith(".py"):
+                legacy_fpath = os.path.join(root, fname)
+                tf_keras_root = root.replace("/_legacy", "/_tf_keras/keras")
+                core_api_fpath = os.path.join(
+                    root.replace("/_legacy", ""), fname
+                )
+                if not os.path.exists(tf_keras_root):
+                    os.makedirs(tf_keras_root)
+                tf_keras_fpath = os.path.join(tf_keras_root, fname)
+                with open(legacy_fpath) as f:
+                    legacy_contents = f.read()
+                    legacy_contents = legacy_contents.replace(
+                        "keras.api._legacy", "keras.api._tf_keras.keras"
+                    )
+                if os.path.exists(core_api_fpath):
+                    with open(core_api_fpath) as f:
+                        core_api_contents = f.read()
+                    core_api_contents = core_api_contents.replace(
+                        "from keras.api import _tf_keras\n", ""
+                    )
+                    for legacy_submodule in legacy_submodules:
+                        core_api_contents = core_api_contents.replace(
+                            f"from keras.api import {legacy_submodule}\n",
+                            "",
+                        )
+                        core_api_contents = core_api_contents.replace(
+                            f"keras.api.{legacy_submodule}",
+                            f"keras.api._tf_keras.keras.{legacy_submodule}",
+                        )
+                    legacy_contents = core_api_contents + "\n" + legacy_contents
+                with open(tf_keras_fpath, "w") as f:
+                    f.write(legacy_contents)
+
+    # Delete keras/api/_legacy/
+    shutil.rmtree(os.path.join(API_DIR, "_legacy"))
+
+
+def export_version_string():
+    API_INIT = os.path.join(package, "api", "__init__.py")
+    with open(API_INIT) as f:
+        contents = f.read()
+    with open(API_INIT, "w") as f:
+        contents += "from keras.src.version import __version__\n"
+        f.write(contents)
+
+
+def update_package_init():
+    contents = """
+# Import everything from /api/ into keras.
+from keras.api import *  # noqa: F403
+from keras.api import __version__  # Import * ignores names start with "_".
+
+import os
+
+# Add everything in /api/ to the module search path.
+__path__.append(os.path.join(os.path.dirname(__file__), "api"))  # noqa: F405
+
+# Don't pollute namespace.
+del os
+
+# Never autocomplete `.src` or `.api` on an imported keras object.
+def __dir__():
+    keys = dict.fromkeys((globals().keys()))
+    keys.pop("src")
+    keys.pop("api")
+    return list(keys)
+
+
+# Don't import `.src` or `.api` during `from keras import *`.
+__all__ = [
+    name
+    for name in globals().keys()
+    if not (name.startswith("_") or name in ("src", "api"))
+]"""
+    with open(os.path.join(package, "__init__.py")) as f:
+        init_contents = f.read()
+    with open(os.path.join(package, "__init__.py"), "w") as f:
+        f.write(init_contents.replace("\nfrom keras import api", contents))
+
+
+if __name__ == "__main__":
+    os.makedirs(os.path.join(package, "api"), exist_ok=True)
+    init_fname = os.path.join(package, "__init__.py")
+    backup_init_fname = os.path.join(package, "__init__.py.bak")
+    shutil.move(init_fname, backup_init_fname)
+    try:
+        namex.generate_api_files(
+            "keras", code_directory="src", target_directory="api"
+        )
+        update_package_init()
+    except Exception as e:
+        shutil.move(backup_init_fname, init_fname)
+        raise e
+    finally:
+        if os.path.exists(backup_init_fname):
+            os.remove(backup_init_fname)
+    export_version_string()
+    create_legacy_directory()

keras/__init__.py

@@ -1,19 +1,33 @@
-from keras import activations
-from keras import applications
-from keras import backend
-from keras import constraints
-from keras import datasets
-from keras import initializers
-from keras import layers
-from keras import models
-from keras import ops
-from keras import optimizers
-from keras import regularizers
-from keras import utils
-from keras.backend import KerasTensor
-from keras.layers import Input
-from keras.layers import Layer
-from keras.models import Functional
-from keras.models import Model
-from keras.models import Sequential
-from keras.version import __version__
+"""DO NOT EDIT.
+
+This file was autogenerated. Do not edit it by hand,
+since your modifications would be overwritten.
+"""
+
+import os
+
+# Import everything from /api/ into keras.
+from keras.api import *  # noqa: F403
+from keras.api import __version__  # Import * ignores names start with "_".
+
+# Add everything in /api/ to the module search path.
+__path__.append(os.path.join(os.path.dirname(__file__), "api"))  # noqa: F405
+
+# Don't pollute namespace.
+del os
+
+
+# Never autocomplete `.src` or `.api` on an imported keras object.
+def __dir__():
+    keys = dict.fromkeys((globals().keys()))
+    keys.pop("src")
+    keys.pop("api")
+    return list(keys)
+
+
+# Don't import `.src` or `.api` during `from keras import *`.
+__all__ = [
+    name
+    for name in globals().keys()
+    if not (name.startswith("_") or name in ("src", "api"))
+]

keras/api/__init__.py

@@ -0,0 +1,57 @@
+"""DO NOT EDIT.
+
+This file was autogenerated. Do not edit it by hand,
+since your modifications would be overwritten.
+"""
+
+from keras.api import _tf_keras
+from keras.api import activations
+from keras.api import applications
+from keras.api import backend
+from keras.api import callbacks
+from keras.api import config
+from keras.api import constraints
+from keras.api import datasets
+from keras.api import distribution
+from keras.api import dtype_policies
+from keras.api import export
+from keras.api import initializers
+from keras.api import layers
+from keras.api import legacy
+from keras.api import losses
+from keras.api import metrics
+from keras.api import mixed_precision
+from keras.api import models
+from keras.api import ops
+from keras.api import optimizers
+from keras.api import preprocessing
+from keras.api import quantizers
+from keras.api import random
+from keras.api import regularizers
+from keras.api import saving
+from keras.api import tree
+from keras.api import utils
+from keras.src.backend.common.keras_tensor import KerasTensor
+from keras.src.backend.common.stateless_scope import StatelessScope
+from keras.src.backend.exports import Variable
+from keras.src.backend.exports import device
+from keras.src.backend.exports import name_scope
+from keras.src.dtype_policies.dtype_policy import DTypePolicy
+from keras.src.dtype_policies.dtype_policy import FloatDTypePolicy
+from keras.src.dtype_policies.dtype_policy import QuantizedDTypePolicy
+from keras.src.initializers.initializer import Initializer
+from keras.src.layers.core.input_layer import Input
+from keras.src.layers.input_spec import InputSpec
+from keras.src.layers.layer import Layer
+from keras.src.losses.loss import Loss
+from keras.src.metrics.metric import Metric
+from keras.src.models.model import Model
+from keras.src.models.sequential import Sequential
+from keras.src.ops.function import Function
+from keras.src.ops.operation import Operation
+from keras.src.optimizers.optimizer import Optimizer
+from keras.src.quantizers.quantizers import AbsMaxQuantizer
+from keras.src.quantizers.quantizers import Quantizer
+from keras.src.regularizers.regularizers import Regularizer
+from keras.src.version import __version__
+from keras.src.version import version

keras/api/_tf_keras/__init__.py

@@ -0,0 +1 @@
+from keras.api._tf_keras import keras