comprehensive error tracking

.gitignore

@@ -6,6 +6,7 @@ examples/research/
 .pytest_cache/
 colour_mnist.py
 flac.py
+.log
 
 # =================== PYCHARM ===================
 # idea

README.md

@@ -24,3 +24,16 @@ Learn more in the [documentation](https://fairbench.readthedocs.io/).
 [<img alt="branches" width="20%" src="docs/images/tabular.png" />](examples/demos/demo.ipynb)
 [<img alt="branches" width="20%" src="docs/images/graphs.png" />](examples/demos/graphs.ipynb)
 [<img alt="branches" width="20%" src="docs/images/vision.png" />](examples/demos/vision.ipynb)
+
+## Citation
+
+```
+@article{krasanakis2024standardizing,
+      title={Towards Standardizing AI Bias Exploration}, 
+      author={Emmanouil Krasanakis and Symeon Papadopoulos},
+      year={2024},
+      eprint={2405.19022},
+      archivePrefix={arXiv},
+      primaryClass={cs.LG}
+}
+```

docs/basics/forks.md

@@ -183,7 +183,7 @@ for multiple attributes with the bitwise *and* `&`.
 
 <div id="code8" class="code-block" style="display:none;">
 ```python
-import fairbench as fbimport fairbench as fb
+import fairbench as fb
 sensitive = fb.Fork(fb.categories@["Man", "Woman", "Man", "Nonbin"]
                     & fb.categories@["Black", "Black", "White", "White"])
 print(sensitive)

docs/basics/interactive.md

@@ -91,7 +91,7 @@ import fairbench as fb
 reportAlg1 = ...  # a report
 reportAlg2 = ...  # a report
 property = ...   # a property (e.g., row or column of the report) to focus on
-reports = {"al1": reportAlg1, "alg2": reportAlg2} # with the dict you can also create the reports incrementally
+reports = {"alg1": reportAlg1, "alg2": reportAlg2} # with the dict you can also create the reports incrementally
 fb = fb.Fork(reports)
 fb.describe(fb[property]) 
 ```
@@ -186,7 +186,7 @@ comprehensive during complex intersectional analysis.
 ```python
 import fairbench as fb
 
-obj = ...  # the object to explore (e.g., a  report)
+obj = ...  # the object to explore (e.g., a report, or fork of reports)
 fb.interactive(obj, name="report", horizontal=True)
 ```
 

docs/index.md

@@ -19,4 +19,18 @@ Visit one of the links below, or read the full documentation.
 
 </div>
 
-<br>
+<br>
+
+
+## How to cite
+
+```
+@article{krasanakis2024standardizing,
+      title={Towards Standardizing AI Bias Exploration}, 
+      author={Emmanouil Krasanakis and Symeon Papadopoulos},
+      year={2024},
+      eprint={2405.19022},
+      archivePrefix={arXiv},
+      primaryClass={cs.LG}
+}
+```

examples/playground/binreport_card.py

@@ -0,0 +1,16 @@
+import fairbench as fb
+import torch
+
+report = fb.multireport(
+    predictions=torch.randint(2, (2,)),
+    labels=torch.randint(2, (2,)),
+    sensetive=fb.Fork(fb.categories @ ["m", "f"]),
+    metrics=[fb.metrics.accuracy, fb.metrics.pr, fb.metrics.fpr, fb.metrics.fnr],
+)
+print(report)
+stamps = fb.combine(
+    fb.stamps.prule(report), fb.stamps.accuracy(report), fb.stamps.four_fifths(report)
+)
+print(fb.stamps.available())
+print(stamps.symbols)
+fb.modelcards.tohtml(stamps, file="model_card.html", show=True)

examples/playground/example.py

@@ -1,23 +1,12 @@
-import pygrank as pg
 import fairbench as fb
 
-"""load data and set sensitive attribute"""
-_, graph, communities = next(pg.load_datasets_multiple_communities(["highschool"]))
-train, test = pg.split(pg.to_signal(graph, communities[0]), 0.5)
-sensitive_signal = pg.to_signal(graph, communities[1])
-labels = test.filter(exclude=train)
-sensitive = fb.Fork(gender=fb.categories@sensitive_signal.filter(exclude=train))
+# testing heterogeneous setting
+test, y, yhat = fb.demos.adult(predict="probabilities")
+s = fb.Fork(fb.categories @ test[9])  # test[8] is a pandas column with race values
 
-"""create report for pagerank"""
-algorithm = pg.PageRank(alpha=0.85)
-scores = algorithm(train).filter(exclude=train)
-report = fb.multireport(labels=labels, scores=scores, sensitive=sensitive)
-
-"""create report for locally fair pagerank"""
-fair_algorithm = pg.LFPR(alpha=0.85, redistributor="original")
-fair_scores = fair_algorithm(train, sensitive=sensitive_signal).filter(exclude=train)
-fair_report = fb.multireport(labels=labels, scores=fair_scores, sensitive=sensitive)
-
-"""combine both reports into one and get the auc perspective"""
-fork = fb.Fork(ppr=report, lfpr=fair_report)
-fb.describe(fork.auc)
+report = fb.multireport(
+    scores=yhat, predictions=(yhat > 0.5), labels=y, sensitive=s, top=50
+)
+# report2 = fb.unireport(predictions=(yhat > 0.5), labels=y, sensitive=s, top=50)
+# report = fb.combine(report, report2)
+# fb.interactive(report)

examples/playground/example_roc.py

@@ -9,4 +9,8 @@
 )
 # report2 = fb.unireport(predictions=(yhat > 0.5), labels=y, sensitive=s, top=50)
 # report = fb.combine(report, report2)
-fb.interactive(report)
+# fb.interactive(report)
+print(report.accuracy.minratio.explain.explain)
+# print(report.auc.maxbarea.explain.explain)
+# fb.visualize(report.auc.maxbarea.explain.explain)
+# fb.visualize(report.avgscore.maxbarea.explain.explain.curve)