[go: nahoru, domu]
Skip to content

Commit

Permalink
MCT 1.0 release
Browse files Browse the repository at this point in the history 
Specifically, this commit...

* updates RELEASE.md and verison.py to 1.0.0
* updates setup.py with explicitly-defined submodule paths (the `find_packages` function does not find the modoel_card_toolkit.proto submodule)
* updates go/mct-release instructions
* updates notebooks to work with schema v2

PiperOrigin-RevId: 388333574
  • Loading branch information
@shuklak13 @ml-fairness-infra-github
shuklak13 authored and ml-fairness-infra-github committed Aug 2, 2021
1 parent 101bac0 commit 0c6ee2b
Show file tree
Hide file tree
Showing 7 changed files with 71 additions and 66 deletions.
2 changes: 1 addition & 1 deletion RELEASE.md
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
<!-- mdlint off(HEADERS_TOO_MANY_H1) -->

# Current Version(Still in Development)
# Release 1.0.0

## Major Features and Improvements

Expand Down
32 changes: 16 additions & 16 deletions model_card_toolkit/documentation/examples/MLMD_Model_Card_Toolkit_Demo.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1080,9 +1080,9 @@
},
"outputs": [],
"source": [
"from model_card_toolkit import ModelCardToolkit\n",
"import model_card_toolkit as mctlib\n",
"\n",
"mct = ModelCardToolkit(mlmd_store=mlmd_store, model_uri=model_uri)"
"mct = mctlib.ModelCardToolkit(mlmd_store=mlmd_store, model_uri=model_uri)"
]
},
{
Expand Down Expand Up @@ -1135,7 +1135,8 @@
" 'about the model’s intended uses, limitations, and ethical considerations.'\n",
")\n",
"model_card.model_details.owners = [\n",
" {'name': 'Model Cards Team', 'contact': 'model-cards@google.com'}\n",
" mctlib.Owner(name='Model Cards Team',\n",
" contact='model-cards@google.com')\n",
"]"
]
},
Expand All @@ -1147,14 +1148,14 @@
},
"outputs": [],
"source": [
"model_card.considerations.use_cases = [\n",
"model_card.considerations.use_cases = [mctlib.UseCase(description=\n",
" 'This dataset that this model was trained on was originally created to '\n",
" 'support the machine learning community in conducting empirical analysis '\n",
" 'of ML algorithms. The Adult Data Set can be used in fairness-related '\n",
" 'studies that compare inequalities across sex and race, based on '\n",
" 'people’s annual incomes.'\n",
" 'people’s annual incomes.')\n",
"]\n",
"model_card.considerations.limitations = [\n",
"model_card.considerations.limitations = [mctlib.Limitation(description=\n",
" 'This is a class-imbalanced dataset across a variety of sensitive classes.'\n",
" ' The ratio of male-to-female examples is about 2:1 and there are far more'\n",
" ' examples with the “white” attribute than every other race combined. '\n",
Expand All @@ -1166,17 +1167,16 @@
" 'fewer female examples in the $50,000+ earner group, causing our model to '\n",
" 'overfit these examples. To avoid this, we can try various remediation '\n",
" 'strategies in future iterations (e.g. undersampling, hyperparameter '\n",
" 'tuning, etc), but we may not be able to fix all of the fairness issues.'\n",
" 'tuning, etc), but we may not be able to fix all of the fairness issues.')\n",
"]\n",
"model_card.considerations.ethical_considerations = [{\n",
" 'name':\n",
" 'We risk expressing the viewpoint that the attributes in this dataset '\n",
" 'are the only ones that are predictive of someone’s income, even '\n",
" 'though we know this is not the case.',\n",
" 'mitigation_strategy':\n",
" 'As mentioned, some interventions may need to be performed to address '\n",
" 'the class imbalances in the dataset.'\n",
"}]"
"model_card.considerations.ethical_considerations = [mctlib.Risk(\n",
" name= 'We risk expressing the viewpoint that the attributes in this dataset '\n",
" 'are the only ones that are predictive of someone’s income, even '\n",
" 'though we know this is not the case.',\n",
" mitigation_strategy= 'As mentioned, some interventions may need to be '\n",
" 'performed to address the class imbalances in the dataset.'\n",
" )\n",
"]"
]
},
{
Expand Down
49 changes: 25 additions & 24 deletions model_card_toolkit/documentation/examples/Scikit_Learn_Model_Card_Toolkit_Demo.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@
"from datetime import date\n",
"from io import BytesIO\n",
"from IPython import display\n",
"from model_card_toolkit import ModelCardToolkit\n",
"import model_card_toolkit as mctlib\n",
"from sklearn.datasets import load_breast_cancer\n",
"from sklearn.ensemble import GradientBoostingClassifier\n",
"from sklearn.model_selection import train_test_split\n",
Expand Down Expand Up @@ -330,7 +330,7 @@
},
"outputs": [],
"source": [
"mct = ModelCardToolkit()\n",
"mct = mctlib.ModelCardToolkit()\n",
"\n",
"model_card = mct.scaffold_assets()"
]
Expand All @@ -357,45 +357,46 @@
" 'This model predicts whether breast cancer is benign or malignant based on '\n",
" 'image measurements.')\n",
"model_card.model_details.owners = [\n",
" {'name': 'Model Cards Team', 'contact': 'model-cards@google.com'}\n",
" mctlib.Owner(name= 'Model Cards Team', contact='model-cards@google.com')\n",
"]\n",
"model_card.model_details.references = [\n",
" 'https://archive.ics.uci.edu/ml/datasets/Breast+Cancer+Wisconsin+(Diagnostic)',\n",
" 'https://minds.wisconsin.edu/bitstream/handle/1793/59692/TR1131.pdf'\n",
" mctlib.Reference(reference='https://archive.ics.uci.edu/ml/datasets/Breast+Cancer+Wisconsin+(Diagnostic)'),\n",
" mctlib.Reference(reference='https://minds.wisconsin.edu/bitstream/handle/1793/59692/TR1131.pdf')\n",
"]\n",
"model_card.model_details.version.name = str(uuid.uuid4())\n",
"model_card.model_details.version.date = str(date.today())\n",
"\n",
"model_card.considerations.ethical_considerations = [{\n",
" 'name': ('Manual selection of image sections to digitize could create '\n",
"model_card.considerations.ethical_considerations = [mctlib.Risk(\n",
" name=('Manual selection of image sections to digitize could create '\n",
" 'selection bias'),\n",
" 'mitigation_strategy': 'Automate the selection process'\n",
"}]\n",
"model_card.considerations.limitations = ['Breast cancer diagnosis']\n",
"model_card.considerations.use_cases = ['Breast cancer diagnosis']\n",
"model_card.considerations.users = ['Medical professionals', 'ML researchers']\n",
" mitigation_strategy='Automate the selection process'\n",
")]\n",
"model_card.considerations.limitations = [mctlib.Limitation(description='Breast cancer diagnosis')]\n",
"model_card.considerations.use_cases = [mctlib.UseCase(description='Breast cancer diagnosis')]\n",
"model_card.considerations.users = [mctlib.User(description='Medical professionals'), User(description='ML researchers')]\n",
"\n",
"\n",
"model_card.model_parameters.data.train.graphics.description = (\n",
"model_card.model_parameters.data.append(mctlib.Dataset())\n",
"model_card.model_parameters.data[0].graphics.description = (\n",
" f'{len(X_train)} rows with {len(X_train.columns)} features')\n",
"model_card.model_parameters.data.train.graphics.collection = [\n",
" {'image': mean_radius_train},\n",
" {'image': mean_texture_train}\n",
"model_card.model_parameters.data[0].graphics.collection = [\n",
" mctlib.Graphic(image=mean_radius_train),\n",
" mctlib.Graphic(image=mean_texture_train)\n",
"]\n",
"model_card.model_parameters.data.eval.graphics.description = (\n",
"model_card.model_parameters.data.append(mctlib.Dataset())\n",
"model_card.model_parameters.data[1].graphics.description = (\n",
" f'{len(X_test)} rows with {len(X_test.columns)} features')\n",
"model_card.model_parameters.data.eval.graphics.collection = [\n",
" {'image': mean_radius_test},\n",
" {'image': mean_texture_test}\n",
"model_card.model_parameters.data[1].graphics.collection = [\n",
" mctlib.Graphic(image=mean_radius_test),\n",
" mctlib.Graphic(image=mean_texture_test)\n",
"]\n",
"model_card.quantitative_analysis.graphics.description = (\n",
" 'ROC curve and confusion matrix')\n",
"model_card.quantitative_analysis.graphics.collection = [\n",
" {'image': roc_curve},\n",
" {'image': confusion_matrix}\n",
" mctlib.Graphic(image=roc_curve),\n",
" mctlib.Graphic(image=confusion_matrix)\n",
"]\n",
"\n",
"mct.update_model_card_json(model_card)"
"mct.update_model_card(model_card)"
]
},
{
Expand Down
43 changes: 22 additions & 21 deletions model_card_toolkit/documentation/examples/Standalone_Model_Card_Toolkit_Demo.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,7 @@
"source": [
"import tensorflow as tf\n",
"import numpy as np\n",
"from model_card_toolkit import ModelCardToolkit\n",
"import model_card_toolkit as mctlib\n",
"from model_card_toolkit.documentation.examples import cats_vs_dogs\n",
"from model_card_toolkit.utils.graphics import figure_to_base64str\n",
"import tempfile\n",
Expand Down Expand Up @@ -335,15 +335,15 @@
" 'performed with high accuracy on both Cat and Dog images.'\n",
")\n",
"model_card.model_details.owners = [\n",
" {'name': 'Model Cards Team', 'contact': 'model-cards@google.com'}\n",
" mctlib.Owner(name='Model Cards Team', contact='model-cards@google.com')\n",
"]\n",
"model_card.model_details.version = {'name': 'v1.0', 'date': '08/28/2020'}\n",
"model_card.model_details.version = mctlib.Version(name='v1.0', date='08/28/2020')\n",
"model_card.model_details.references = [\n",
" 'https://www.tensorflow.org/guide/keras/transfer_learning',\n",
" 'https://arxiv.org/abs/1801.04381',\n",
" mctlib.Reference(reference='https://www.tensorflow.org/guide/keras/transfer_learning'),\n",
" mctlib.Reference(reference='https://arxiv.org/abs/1801.04381'),\n",
"]\n",
"model_card.model_details.license = 'Apache-2.0'\n",
"model_card.model_details.citation = 'https://github.com/tensorflow/model-card-toolkit/blob/master/model_card_toolkit/documentation/examples/Standalone_Model_Card_Toolkit_Demo.ipynb'"
"model_card.model_details.licenses = [mctlib.License(identifier='Apache-2.0')]\n",
"model_card.model_details.citations = [mctlib.Citation(citation='https://github.com/tensorflow/model-card-toolkit/blob/master/model_card_toolkit/documentation/examples/Standalone_Model_Card_Toolkit_Demo.ipynb')]"
]
},
{
Expand All @@ -368,9 +368,9 @@
"outputs": [],
"source": [
"model_card.quantitative_analysis.performance_metrics = [\n",
" {'type': 'accuracy', 'value': accuracy},\n",
" {'type': 'accuracy', 'value': cat_accuracy, 'slice': 'cat'},\n",
" {'type': 'accuracy', 'value': dog_accuracy, 'slice': 'Dog'},\n",
" mctlib.PerformanceMetric(type='accuracy', value=str(accuracy)),\n",
" mctlib.PerformanceMetric(type='accuracy', value=str(cat_accuracy), slice='cat'),\n",
" mctlib.PerformanceMetric(type='accuracy', value=str(dog_accuracy), slice='Dog'),\n",
"]"
]
},
Expand All @@ -394,20 +394,20 @@
"outputs": [],
"source": [
"model_card.considerations.use_cases = [\n",
" 'This model classifies images of cats and dogs.'\n",
" mctlib.UseCase(description='This model classifies images of cats and dogs.')\n",
"]\n",
"model_card.considerations.limitations = [\n",
" 'This model is not able to classify images of other classes.'\n",
" mctlib.Limitation(description='This model is not able to classify images of other classes.')\n",
"]\n",
"model_card.considerations.ethical_considerations = [{\n",
" 'name':\n",
"model_card.considerations.ethical_considerations = [mctlib.Risk(\n",
" name=\n",
" 'While distinguishing between cats and dogs is generally agreed to be '\n",
" 'a benign application of machine learning, harmful results can occur '\n",
" 'when the model attempts to classify images that don’t contain cats or '\n",
" 'dogs.',\n",
" 'mitigation_strategy':\n",
" mitigation_strategy=\n",
" 'Avoid application on non-dog and non-cat images.'\n",
"}]"
")]"
]
},
{
Expand Down Expand Up @@ -486,11 +486,12 @@
},
"outputs": [],
"source": [
"model_card.model_parameters.data.eval.graphics.collection = [\n",
" {'name': 'Validation Set Size', 'image': validation_set_size_barchart},\n",
"model_card.model_parameters.data.append(mctlib.Dataset())\n",
"model_card.model_parameters.data[0].graphics.collection = [\n",
" mctlib.Graphic(name='Validation Set Size', image=validation_set_size_barchart),\n",
"]\n",
"model_card.quantitative_analysis.graphics.collection = [\n",
" {'name': 'Accuracy', 'image': accuracy_barchart},\n",
" mctlib.Graphic(name='Accuracy', image=accuracy_barchart),\n",
"]"
]
},
Expand All @@ -514,7 +515,7 @@
},
"outputs": [],
"source": [
"mct.update_model_card_json(model_card)"
"mct.update_model_card(model_card)"
]
},
{
Expand Down Expand Up @@ -560,7 +561,7 @@
"source": [
"# Generate a model card document in Markdown\n",
"md_path = os.path.join(model_card_dir, 'template/md/default_template.md.jinja')\n",
"md_doc = mct.export_format(md_path, 'model_card.md')\n",
"md_doc = mct.export_format(template_path=md_path, 'model_card.md')\n",
"\n",
"# Display the model card document in Markdown\n",
"display.display(display.Markdown(md_doc))"
Expand Down
2 changes: 1 addition & 1 deletion model_card_toolkit/model_card.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,10 @@
ModelCardsToolkit serves as an API to read and write MC properties by the users.
"""

import dataclasses
import json as json_lib
from typing import Any, Dict, List, Optional, Text

import dataclasses
from model_card_toolkit.base_model_card_field import BaseModelCardField
from model_card_toolkit.proto import model_card_pb2
from model_card_toolkit.utils import validation
Expand Down
2 changes: 1 addition & 1 deletion model_card_toolkit/version.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,4 +14,4 @@
"""Contains the version string of the Model Card Toolkit."""

# Note that setup.py uses this version.
__version__ = '0.1.3.dev'
__version__ = '1.0.0'
7 changes: 5 additions & 2 deletions setup.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@
import subprocess

from setuptools import Command
from setuptools import find_packages
from setuptools import setup

REQUIRED_PACKAGES = [
Expand Down Expand Up @@ -110,7 +109,11 @@ def run(self):
url='https://github.com/tensorflow/model-card-toolkit',
author='Google LLC',
author_email='tensorflow-extended-dev@googlegroups.com',
packages=find_packages(exclude=('bazel-model_card_toolkit*',)),
packages=[
'model_card_toolkit', 'model_card_toolkit.documentation',
'model_card_toolkit.documentation.examples', 'model_card_toolkit.proto',
'model_card_toolkit.utils'
],
package_data={
'model_card_toolkit': ['schema/**/*.json', 'template/**/*.jinja']
},
Expand Down

0 comments on commit 0c6ee2b

Please sign in to comment.