End-to-End Multimodal Data Augmentation and Adversarial Robustness Benchmark with AugLy for Images, Text, Audio, and PyTorch
import subprocess, sys, importlib
def _sh(cmd):
print(f”$ {cmd}”)
subprocess.run(cmd, shell=True, check=False,
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
def _need(mod):
try:
importlib.import_module(mod)
return False
except ImportError:
return True
if _need(“augly”):
_sh(“apt-get -qq install -y libmagic1 > /dev/null 2>&1″)
_sh(f'”{sys.executable}” -m pip install -q –no-deps augly’)
_sh(f'”{sys.executable}” -m pip install -q “iopath>=0.1.8” “python-magic>=0.4.22″ ‘
f'”regex>=2021.4.4” “nlpaug==1.1.3″‘)
import numpy as np
from PIL import Image, ImageDraw, ImageFont, ImageFilter
for _name, _builtin in ((“float”, float), (“int”, int), (“bool”, bool)):
if not hasattr(np, _name):
setattr(np, _name, _builtin)
def _size(font, text):
left, top, right, bottom = font.getbbox(text)
return (right, bottom)
if not hasattr(ImageFont.FreeTypeFont, “getsize”):
ImageFont.FreeTypeFont.getsize = lambda self, t, *a, **k: _size(self, t)
if not hasattr(ImageFont.FreeTypeFont, “getsize_multiline”):
def _getsize_multiline(self, text, direction=None, spacing=4, features=None,
language=None, stroke_width=0):
lines = text.split(“\n”)
w = max((_size(self, ln)[0] for ln in lines), default=0)
h = sum(_size(self, ln)[1] for ln in lines) + spacing * (len(lines) – 1)
return (w, h)
ImageFont.FreeTypeFont.getsize_multiline = _getsize_multiline
import os, io, json, math, random, string, textwrap, unicodedata, warnings
from dataclasses import dataclass
from typing import Any, Dict, List, Optional, Tuple
import matplotlib.pyplot as plt
import pandas as pd
import augly.image as imaugs
import augly.text as textaugs
import augly.utils as augutils
from augly.image.transforms import BaseTransform as ImageBaseTransform
warnings.filterwarnings(“ignore”)
pd.set_option(“display.width”, 160)
SEED = 1234
random.seed(SEED)
np.random.seed(SEED)
print(“\n” + “=” * 78)
print(“AugLy ready. assets at:”, augutils.ASSETS_BASE_DIR)
print(“image augs :”, len([f for f in dir(imaugs) if f[0].islower()]))
print(“text augs :”, len([f for f in dir(textaugs) if f[0].islower()]))
print(“=” * 78 + “\n”)
def make_image(idx: int, w: int = 320, h: int = 240) -> Tuple[Image.Image, Tuple[int, int, int, int]]:
“””Procedurally generated ‘photo’ + a ground-truth bbox in pascal_voc format.”””
rng = random.Random(SEED + idx)
img = Image.new(“RGB”, (w, h), tuple(rng.randint(20, 90) for _ in range(3)))
d = ImageDraw.Draw(img)
for _ in range(70):
x0, y0 = rng.randint(0, w), rng.randint(0, h)
d.line([x0, y0, x0 + rng.randint(-60, 60), y0 + rng.randint(-60, 60)],
fill=tuple(rng.randint(60, 160) for _ in range(3)), width=rng.randint(1, 3))
ow, oh = rng.randint(70, 130), rng.randint(60, 110)
ox, oy = rng.randint(10, w – ow – 10), rng.randint(10, h – oh – 10)
box = (ox, oy, ox + ow, oy + oh)
colour = tuple(rng.randint(150, 255) for _ in range(3))
if idx % 3 == 0:
d.ellipse(box, fill=colour, outline=(255, 255, 255), width=3)
elif idx % 3 == 1:
d.rectangle(box, fill=colour, outline=(255, 255, 255), width=3)
else:
d.polygon([(ox + ow // 2, oy), (ox + ow, oy + oh), (ox, oy + oh)],
fill=colour, outline=(255, 255, 255))
return img, box
N_IMAGES = 24
IMAGES, BOXES = zip(*[make_image(i) for i in range(N_IMAGES)])
IMAGES, BOXES = list(IMAGES), list(BOXES)
DEMO_IMG, DEMO_BOX = IMAGES[0], BOXES[0]
def make_text_dataset(n_per_class: int = 260):
“””Tiny sentiment corpus built from templates -> learnable but not trivial.”””
rng = random.Random(SEED)
pos_adj = [“excellent”, “delightful”, “superb”, “charming”, “brilliant”,
“flawless”, “wonderful”, “outstanding”, “impressive”, “lovely”]
neg_adj = [“terrible”, “awful”, “dreadful”, “disappointing”, “clumsy”,
“broken”, “miserable”, “useless”, “painful”, “sloppy”]
subj = [“the movie”, “this restaurant”, “the hotel room”, “their support team”,
“the new phone”, “the sequel”, “this laptop”, “the delivery service”]
tail_p = [“and I would recommend it to anyone”, “worth every rupee”,
“I left completely satisfied”, “easily the best of the year”,
“it exceeded all my expectations”]
tail_n = [“and I want a refund”, “a total waste of money”,
“I left extremely frustrated”, “easily the worst of the year”,
“it failed every expectation”]
rows = []
for _ in range(n_per_class):
rows.append((f”{rng.choice(subj)} was {rng.choice(pos_adj)} {rng.choice(tail_p)}”, 1))
rows.append((f”{rng.choice(subj)} was {rng.choice(neg_adj)} {rng.choice(tail_n)}”, 0))
rng.shuffle(rows)
return [r[0] for r in rows], [r[1] for r in rows]
TEXTS, LABELS = make_text_dataset()
DEMO_TEXT = “The quick brown fox jumps over the lazy dog near the river bank”
def make_audio(seconds: float = 2.0, sr: int = 16000) -> Tuple[np.ndarray, int]:
“””A chirp + harmonics + a little noise = something you can actually hear change.”””
t = np.linspace(0, seconds, int(sr * seconds), endpoint=False)
f = np.linspace(220, 880, t.size)
sig = 0.5 * np.sin(2 * np.pi * f * t) + 0.2 * np.sin(2 * np.pi * 2 * f * t)
sig += 0.02 * np.random.RandomState(SEED).randn(t.size)
env = np.minimum(1.0, np.minimum(t * 8, (seconds – t) * 8))
return (sig * env).astype(np.float32), sr
AUDIO, SR = make_audio()
def show_grid(pairs, cols=4, title=””, figsize_scale=2.9):
“””pairs: list of (caption, PIL.Image).”””
rows = math.ceil(len(pairs) / cols)
fig, axes = plt.subplots(rows, cols, figsize=(cols * figsize_scale, rows * figsize_scale))
axes = np.atleast_1d(axes).ravel()
for ax, (cap, im) in zip(axes, pairs):
ax.imshow(im)
ax.set_title(cap, fontsize=8)
ax.axis(“off”)
for ax in axes[len(pairs):]:
ax.axis(“off”)
if title:
fig.suptitle(title, fontsize=13, y=1.0)
plt.tight_layout()
plt.show()
def as_str(out) -> str:
“””AugLy text augs return str for str input in some transforms, list in others.”””
return out[0] if isinstance(out, list) else out
print(“\n### §2 IMAGE AUGMENTATION + METADATA ” + “#” * 38)
functional_result = imaugs.pixelization(DEMO_IMG, ratio=0.25)
class_result = imaugs.Pixelization(ratio=0.25, p=1.0)(DEMO_IMG)
print(“functional == class:”, np.array_equal(np.array(functional_result), np.array(class_result)))
IMAGE_ZOO = {
“blur”: lambda im, m: imaugs.blur(im, radius=3.0, metadata=m),
“brightness”: lambda im, m: imaugs.brightness(im, factor=1.7, metadata=m),
“color_jitter”: lambda im, m: imaugs.color_jitter(im, brightness_factor=1.3,
contrast_factor=1.4,
saturation_factor=1.6, metadata=m),
“crop”: lambda im, m: imaugs.crop(im, x1=.15, y1=.15, x2=.85, y2=.85, metadata=m),
“encoding_quality”: lambda im, m: imaugs.encoding_quality(im, quality=8, metadata=m),
“grayscale”: lambda im, m: imaugs.grayscale(im, metadata=m),
“hflip”: lambda im, m: imaugs.hflip(im, metadata=m),
“meme_format”: lambda im, m: imaugs.meme_format(im, text=”TOP TEXT”,
caption_height=90, metadata=m),
“opacity”: lambda im, m: imaugs.opacity(im, level=0.45, metadata=m),
“overlay_emoji”: lambda im, m: imaugs.overlay_emoji(im, opacity=0.9,
emoji_size=0.35, metadata=m),
“overlay_screenshot”: lambda im, m: imaugs.overlay_onto_screenshot(im, metadata=m),
“overlay_stripes”: lambda im, m: imaugs.overlay_stripes(im, line_width=0.4,
line_opacity=0.7, metadata=m),
“overlay_text”: lambda im, m: imaugs.overlay_text(im, opacity=0.9, metadata=m),
“pad_square”: lambda im, m: imaugs.pad_square(im, metadata=m),
“perspective”: lambda im, m: imaugs.perspective_transform(im, sigma=40.0, metadata=m),
“pixelization”: lambda im, m: imaugs.pixelization(im, ratio=0.15, metadata=m),
“random_noise”: lambda im, m: imaugs.random_noise(im, var=0.03, metadata=m),
“rotate”: lambda im, m: imaugs.rotate(im, degrees=17, metadata=m),
“saturation”: lambda im, m: imaugs.saturation(im, factor=3.0, metadata=m),
“scale”: lambda im, m: imaugs.scale(im, factor=0.35, metadata=m),
“sharpen”: lambda im, m: imaugs.sharpen(im, factor=8.0, metadata=m),
“shuffle_pixels”: lambda im, m: imaugs.shuffle_pixels(im, factor=0.15, metadata=m),
“skew”: lambda im, m: imaugs.skew(im, skew_factor=0.35, metadata=m),
“vflip”: lambda im, m: imaugs.vflip(im, metadata=m),
}
gallery, image_meta = [(“ORIGINAL”, DEMO_IMG)], []
for name, fn in IMAGE_ZOO.items():
m = []
try:
out = fn(DEMO_IMG, m)
gallery.append((f”{name}\nintensity={m[0][‘intensity’]:.1f}”, out))
image_meta.append(m[0])
except Exception as e:
print(f” [skip] {name}: {type(e).__name__}: {e}”)
show_grid(gallery, cols=5, title=”§2 AugLy image augmentations (with AugLy’s own intensity score)”)
meta_df = pd.DataFrame(image_meta)[[“name”, “intensity”, “src_width”, “src_height”,
“dst_width”, “dst_height”]]
print(meta_df.sort_values(“intensity”, ascending=False).head(10).to_string(index=False))
def _sh(cmd):
print(f”$ {cmd}”)
subprocess.run(cmd, shell=True, check=False,
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
def _need(mod):
try:
importlib.import_module(mod)
return False
except ImportError:
return True
if _need(“augly”):
_sh(“apt-get -qq install -y libmagic1 > /dev/null 2>&1″)
_sh(f'”{sys.executable}” -m pip install -q –no-deps augly’)
_sh(f'”{sys.executable}” -m pip install -q “iopath>=0.1.8” “python-magic>=0.4.22″ ‘
f'”regex>=2021.4.4” “nlpaug==1.1.3″‘)
import numpy as np
from PIL import Image, ImageDraw, ImageFont, ImageFilter
for _name, _builtin in ((“float”, float), (“int”, int), (“bool”, bool)):
if not hasattr(np, _name):
setattr(np, _name, _builtin)
def _size(font, text):
left, top, right, bottom = font.getbbox(text)
return (right, bottom)
if not hasattr(ImageFont.FreeTypeFont, “getsize”):
ImageFont.FreeTypeFont.getsize = lambda self, t, *a, **k: _size(self, t)
if not hasattr(ImageFont.FreeTypeFont, “getsize_multiline”):
def _getsize_multiline(self, text, direction=None, spacing=4, features=None,
language=None, stroke_width=0):
lines = text.split(“\n”)
w = max((_size(self, ln)[0] for ln in lines), default=0)
h = sum(_size(self, ln)[1] for ln in lines) + spacing * (len(lines) – 1)
return (w, h)
ImageFont.FreeTypeFont.getsize_multiline = _getsize_multiline
import os, io, json, math, random, string, textwrap, unicodedata, warnings
from dataclasses import dataclass
from typing import Any, Dict, List, Optional, Tuple
import matplotlib.pyplot as plt
import pandas as pd
import augly.image as imaugs
import augly.text as textaugs
import augly.utils as augutils
from augly.image.transforms import BaseTransform as ImageBaseTransform
warnings.filterwarnings(“ignore”)
pd.set_option(“display.width”, 160)
SEED = 1234
random.seed(SEED)
np.random.seed(SEED)
print(“\n” + “=” * 78)
print(“AugLy ready. assets at:”, augutils.ASSETS_BASE_DIR)
print(“image augs :”, len([f for f in dir(imaugs) if f[0].islower()]))
print(“text augs :”, len([f for f in dir(textaugs) if f[0].islower()]))
print(“=” * 78 + “\n”)
def make_image(idx: int, w: int = 320, h: int = 240) -> Tuple[Image.Image, Tuple[int, int, int, int]]:
“””Procedurally generated ‘photo’ + a ground-truth bbox in pascal_voc format.”””
rng = random.Random(SEED + idx)
img = Image.new(“RGB”, (w, h), tuple(rng.randint(20, 90) for _ in range(3)))
d = ImageDraw.Draw(img)
for _ in range(70):
x0, y0 = rng.randint(0, w), rng.randint(0, h)
d.line([x0, y0, x0 + rng.randint(-60, 60), y0 + rng.randint(-60, 60)],
fill=tuple(rng.randint(60, 160) for _ in range(3)), width=rng.randint(1, 3))
ow, oh = rng.randint(70, 130), rng.randint(60, 110)
ox, oy = rng.randint(10, w – ow – 10), rng.randint(10, h – oh – 10)
box = (ox, oy, ox + ow, oy + oh)
colour = tuple(rng.randint(150, 255) for _ in range(3))
if idx % 3 == 0:
d.ellipse(box, fill=colour, outline=(255, 255, 255), width=3)
elif idx % 3 == 1:
d.rectangle(box, fill=colour, outline=(255, 255, 255), width=3)
else:
d.polygon([(ox + ow // 2, oy), (ox + ow, oy + oh), (ox, oy + oh)],
fill=colour, outline=(255, 255, 255))
return img, box
N_IMAGES = 24
IMAGES, BOXES = zip(*[make_image(i) for i in range(N_IMAGES)])
IMAGES, BOXES = list(IMAGES), list(BOXES)
DEMO_IMG, DEMO_BOX = IMAGES[0], BOXES[0]
def make_text_dataset(n_per_class: int = 260):
“””Tiny sentiment corpus built from templates -> learnable but not trivial.”””
rng = random.Random(SEED)
pos_adj = [“excellent”, “delightful”, “superb”, “charming”, “brilliant”,
“flawless”, “wonderful”, “outstanding”, “impressive”, “lovely”]
neg_adj = [“terrible”, “awful”, “dreadful”, “disappointing”, “clumsy”,
“broken”, “miserable”, “useless”, “painful”, “sloppy”]
subj = [“the movie”, “this restaurant”, “the hotel room”, “their support team”,
“the new phone”, “the sequel”, “this laptop”, “the delivery service”]
tail_p = [“and I would recommend it to anyone”, “worth every rupee”,
“I left completely satisfied”, “easily the best of the year”,
“it exceeded all my expectations”]
tail_n = [“and I want a refund”, “a total waste of money”,
“I left extremely frustrated”, “easily the worst of the year”,
“it failed every expectation”]
rows = []
for _ in range(n_per_class):
rows.append((f”{rng.choice(subj)} was {rng.choice(pos_adj)} {rng.choice(tail_p)}”, 1))
rows.append((f”{rng.choice(subj)} was {rng.choice(neg_adj)} {rng.choice(tail_n)}”, 0))
rng.shuffle(rows)
return [r[0] for r in rows], [r[1] for r in rows]
TEXTS, LABELS = make_text_dataset()
DEMO_TEXT = “The quick brown fox jumps over the lazy dog near the river bank”
def make_audio(seconds: float = 2.0, sr: int = 16000) -> Tuple[np.ndarray, int]:
“””A chirp + harmonics + a little noise = something you can actually hear change.”””
t = np.linspace(0, seconds, int(sr * seconds), endpoint=False)
f = np.linspace(220, 880, t.size)
sig = 0.5 * np.sin(2 * np.pi * f * t) + 0.2 * np.sin(2 * np.pi * 2 * f * t)
sig += 0.02 * np.random.RandomState(SEED).randn(t.size)
env = np.minimum(1.0, np.minimum(t * 8, (seconds – t) * 8))
return (sig * env).astype(np.float32), sr
AUDIO, SR = make_audio()
def show_grid(pairs, cols=4, title=””, figsize_scale=2.9):
“””pairs: list of (caption, PIL.Image).”””
rows = math.ceil(len(pairs) / cols)
fig, axes = plt.subplots(rows, cols, figsize=(cols * figsize_scale, rows * figsize_scale))
axes = np.atleast_1d(axes).ravel()
for ax, (cap, im) in zip(axes, pairs):
ax.imshow(im)
ax.set_title(cap, fontsize=8)
ax.axis(“off”)
for ax in axes[len(pairs):]:
ax.axis(“off”)
if title:
fig.suptitle(title, fontsize=13, y=1.0)
plt.tight_layout()
plt.show()
def as_str(out) -> str:
“””AugLy text augs return str for str input in some transforms, list in others.”””
return out[0] if isinstance(out, list) else out
print(“\n### §2 IMAGE AUGMENTATION + METADATA ” + “#” * 38)
functional_result = imaugs.pixelization(DEMO_IMG, ratio=0.25)
class_result = imaugs.Pixelization(ratio=0.25, p=1.0)(DEMO_IMG)
print(“functional == class:”, np.array_equal(np.array(functional_result), np.array(class_result)))
IMAGE_ZOO = {
“blur”: lambda im, m: imaugs.blur(im, radius=3.0, metadata=m),
“brightness”: lambda im, m: imaugs.brightness(im, factor=1.7, metadata=m),
“color_jitter”: lambda im, m: imaugs.color_jitter(im, brightness_factor=1.3,
contrast_factor=1.4,
saturation_factor=1.6, metadata=m),
“crop”: lambda im, m: imaugs.crop(im, x1=.15, y1=.15, x2=.85, y2=.85, metadata=m),
“encoding_quality”: lambda im, m: imaugs.encoding_quality(im, quality=8, metadata=m),
“grayscale”: lambda im, m: imaugs.grayscale(im, metadata=m),
“hflip”: lambda im, m: imaugs.hflip(im, metadata=m),
“meme_format”: lambda im, m: imaugs.meme_format(im, text=”TOP TEXT”,
caption_height=90, metadata=m),
“opacity”: lambda im, m: imaugs.opacity(im, level=0.45, metadata=m),
“overlay_emoji”: lambda im, m: imaugs.overlay_emoji(im, opacity=0.9,
emoji_size=0.35, metadata=m),
“overlay_screenshot”: lambda im, m: imaugs.overlay_onto_screenshot(im, metadata=m),
“overlay_stripes”: lambda im, m: imaugs.overlay_stripes(im, line_width=0.4,
line_opacity=0.7, metadata=m),
“overlay_text”: lambda im, m: imaugs.overlay_text(im, opacity=0.9, metadata=m),
“pad_square”: lambda im, m: imaugs.pad_square(im, metadata=m),
“perspective”: lambda im, m: imaugs.perspective_transform(im, sigma=40.0, metadata=m),
“pixelization”: lambda im, m: imaugs.pixelization(im, ratio=0.15, metadata=m),
“random_noise”: lambda im, m: imaugs.random_noise(im, var=0.03, metadata=m),
“rotate”: lambda im, m: imaugs.rotate(im, degrees=17, metadata=m),
“saturation”: lambda im, m: imaugs.saturation(im, factor=3.0, metadata=m),
“scale”: lambda im, m: imaugs.scale(im, factor=0.35, metadata=m),
“sharpen”: lambda im, m: imaugs.sharpen(im, factor=8.0, metadata=m),
“shuffle_pixels”: lambda im, m: imaugs.shuffle_pixels(im, factor=0.15, metadata=m),
“skew”: lambda im, m: imaugs.skew(im, skew_factor=0.35, metadata=m),
“vflip”: lambda im, m: imaugs.vflip(im, metadata=m),
}
gallery, image_meta = [(“ORIGINAL”, DEMO_IMG)], []
for name, fn in IMAGE_ZOO.items():
m = []
try:
out = fn(DEMO_IMG, m)
gallery.append((f”{name}\nintensity={m[0][‘intensity’]:.1f}”, out))
image_meta.append(m[0])
except Exception as e:
print(f” [skip] {name}: {type(e).__name__}: {e}”)
show_grid(gallery, cols=5, title=”§2 AugLy image augmentations (with AugLy’s own intensity score)”)
meta_df = pd.DataFrame(image_meta)[[“name”, “intensity”, “src_width”, “src_height”,
“dst_width”, “dst_height”]]
print(meta_df.sort_values(“intensity”, ascending=False).head(10).to_string(index=False))
