Add application file

.gitattributes

@@ -33,3 +33,14 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text
+samples/*.png filter=lfs diff=lfs merge=lfs -text
+samples/cable.png filter=lfs diff=lfs merge=lfs -text
+samples/capsule.png filter=lfs diff=lfs merge=lfs -text
+samples/carpet.png filter=lfs diff=lfs merge=lfs -text
+samples/hazelnut.png filter=lfs diff=lfs merge=lfs -text
+samples/leather.png filter=lfs diff=lfs merge=lfs -text
+samples/tile.png filter=lfs diff=lfs merge=lfs -text
+samples/toothbrush.png filter=lfs diff=lfs merge=lfs -text
+samples/transistor.png filter=lfs diff=lfs merge=lfs -text
+samples/wood.png filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,88 @@
+*.txt.user*
+build/*
+dbg_build/*
+# .env is personal
+.env
+
+# various emacs ignore
+*~
+[#]*[#]
+.\#*
+
+# devcontainer vcode
+
+.devcontainer/tmp/*
+
+# Python
+results/*
+**/__pycache__
+.idea/*
+mv
+
+
+
+#CMake Outputs
+CMakeLists.txt.user
+CMakeCache.txt
+CMakeFiles
+CMakeScripts
+Testing
+Makefile
+cmake_install.cmake
+install_manifest.txt
+compile_commands.json
+CTestTestfile.cmake
+_deps
+# C++ objects and libs
+*.slo
+*.lo
+*.o
+*.a
+*.la
+*.lai
+*.so
+*.so.*
+*.dll
+*.dylib
+
+# Qt-es
+object_script.*.Release
+object_script.*.Debug
+*_plugin_import.cpp
+/.qmake.cache
+/.qmake.stash
+*.pro.user
+*.pro.user.*
+*.qbs.user
+*.qbs.user.*
+*.moc
+moc_*.cpp
+moc_*.h
+qrc_*.cpp
+ui_*.h
+*.qmlc
+*.jsc
+Makefile*
+*build-*
+*.qm
+*.prl
+
+# Qt unit tests
+target_wrapper.*
+
+# QtCreator
+*.autosave
+
+# QtCreator Qml
+*.qmlproject.user
+*.qmlproject.user.*
+# QtCreator CMake
+CMakeLists.txt.user*
+
+# QtCreator 4.8< compilation database
+compile_commands.json
+
+# QtCreator local machine specific files for imported projects
+*creator.user*
+*_qmlcache.qrc
+

app.py

@@ -0,0 +1,73 @@
+from source.defectGenerator import DefectGenerator
+import matplotlib.pyplot as plt
+from PIL import Image
+import gradio as gr
+import numpy as np
+
+def generate_defect_image(image, defect_type,category):
+    defGen=DefectGenerator(image.size,dtd_path="samples/dtd/")
+    defect,msk=defGen.genDefect(image,[defect_type],category.lower())
+    defect=(defect.permute(1,2,0).numpy()*255.0).astype('uint8') 
+    msk=(msk.permute(1,2,0).numpy()*255.0).astype('uint8')
+    msk = np.concatenate((msk, msk, msk), axis=2)
+    return defect, msk
+
+images = {
+    "Bottle": Image.open('samples/bottle.png').convert('RGB').resize((1024, 1024)),
+    "Cable": Image.open('samples/cable.png').convert('RGB').resize((1024, 1024)),
+    "Capsule": Image.open('samples/capsule.png').convert('RGB').resize((1024, 1024)),
+    "Carpet": Image.open('samples/carpet.png').convert('RGB').resize((1024, 1024)),
+    "Grid": Image.open('samples/grid.png').convert('RGB').resize((1024, 1024)),
+    "Hazelnut": Image.open('samples/hazelnut.png').convert('RGB').resize((1024, 1024)),
+    "Leather": Image.open('samples/leather.png').convert('RGB').resize((1024, 1024)),
+    "Metal Nut": Image.open('samples/metal_nut.png').convert('RGB').resize((1024, 1024)),
+    "Pill": Image.open('samples/pill.png').convert('RGB').resize((1024, 1024)),
+    "Screw": Image.open('samples/screw.png').convert('RGB').resize((1024, 1024)),
+    "Tile": Image.open('samples/tile.png').convert('RGB').resize((1024, 1024)),
+    "Toothbrush": Image.open('samples/toothbrush.png').convert('RGB').resize((1024, 1024)),
+    "Transistor": Image.open('samples/transistor.png').convert('RGB').resize((1024, 1024)),
+    "Wood": Image.open('samples/wood.png').convert('RGB').resize((1024, 1024)),
+    "Zipper": Image.open('samples/zipper.png').convert('RGB').resize((1024, 1024))
+    
+}
+
+
+def generate_and_display_images(category, defect_type):
+    base_image = images[category]
+    img_with_defect, defect_mask = generate_defect_image(base_image, defect_type,category)
+    return np.array(base_image), img_with_defect, defect_mask
+
+# Components 
+with gr.Blocks(css="style.css") as demo:
+    gr.HTML(
+        "<h1><center> &#127981; MVTEC AD Defect Generator &#127981; </center></h1>" +
+        "<p><center><a href='https://github.com/SimonThomine/IndustrialDefectLib'>https://github.com/SimonThomine/IndustrialDefectLib</a></center></p>"
+    )
+    with gr.Group():
+        with gr.Row():
+            category_input = gr.Dropdown(label="Select object", choices=list(images.keys()),value="Bottle")
+            defect_type_input = gr.Dropdown(label="Select type of defect", choices=["blurred", "nsa","structural", "textural" ],value="nsa")
+            submit = gr.Button(
+                scale=1,
+                variant='primary'
+            )        
+        with gr.Row():
+            with gr.Column(scale=1, min_width=400):
+                gr.HTML("<h1><center> Base </center></h1>")
+                base_image_output = gr.Image("Base", type="numpy")
+                
+            with gr.Column(scale=1, min_width=400):
+                gr.HTML("<h1><center> Mask </center></h1>")
+                mask_output = gr.Image("Mask", type="numpy")
+                
+            with gr.Column(scale=1, min_width=400):
+                gr.HTML("<h1><center> Defect </center></h1>")
+                defect_image_output = gr.Image("Defect", type="numpy")
+                
+    submit.click(
+        fn=generate_and_display_images,
+        inputs=[category_input, defect_type_input],
+        outputs=[base_image_output, defect_image_output,mask_output],
+    )
+    
+demo.launch()

requirements.txt

@@ -0,0 +1,12 @@
+matplotlib
+numpy
+opencv-python
+pandas
+pillow
+scikit-image
+scikit-learn
+timm
+torch
+torchvision
+tqdm
+imgaug

source/defectGenerator.py

@@ -0,0 +1,202 @@
+import torch
+import numpy as np
+import cv2
+import imgaug.augmenters as iaa
+import random
+import torchvision.transforms as T
+import glob
+from source.perlin import rand_perlin_2d_np
+import matplotlib.pyplot as plt
+from source.nsa import backGroundMask,patch_ex
+
+class TexturalAnomalyGenerator():
+    def __init__(self, resize_shape=None,dtd_path="../../datasets/dtd/images"):
+        
+        
+        self.resize_shape=resize_shape
+        self.anomaly_source_paths = sorted(glob.glob(dtd_path+"/*/*.jpg"))
+        self.augmenters = [iaa.GammaContrast((0.5,2.0),per_channel=True),
+                      iaa.MultiplyAndAddToBrightness(mul=(0.8,1.2),add=(-30,30)),
+                      iaa.pillike.EnhanceSharpness(),
+                      iaa.AddToHueAndSaturation((-10,10),per_channel=True),
+                      iaa.Solarize(0.5, threshold=(32,128)),
+                      iaa.Posterize(),
+                      iaa.Invert(),
+                      iaa.pillike.Autocontrast(),
+                      iaa.pillike.Equalize(),
+                      ]
+        
+    
+    def randAugmenter(self):
+        aug_ind = np.random.choice(np.arange(len(self.augmenters)), 3, replace=False)
+        aug = iaa.Sequential([self.augmenters[aug_ind[0]],
+                              self.augmenters[aug_ind[1]],
+                              self.augmenters[aug_ind[2]]]
+                             )
+        return aug
+    def getDtdImage(self):
+        randIndex=random.randint(0, len(self.anomaly_source_paths)-1)
+        image=cv2.imread(self.anomaly_source_paths[randIndex])
+        image=cv2.resize(image, dsize=(self.resize_shape[0], self.resize_shape[1]))
+        aug=self.randAugmenter()
+        image=aug(image=image)
+        return image 
+    
+
+class StructuralAnomalyGenerator():
+    def __init__(self,resize_shape=None):
+        
+        self.resize_shape=resize_shape
+        self.augmenters = [iaa.Fliplr(0.5),  
+                            iaa.Affine(rotate=(-45, 45)),  
+                            iaa.Multiply((0.8, 1.2)),  
+                            iaa.MultiplySaturation((0.5, 1.5)), 
+                            iaa.MultiplyHue((0.5, 1.5))
+                      ]
+    
+    def randAugmenter(self):
+        aug_ind = np.random.choice(np.arange(len(self.augmenters)), 3, replace=False)
+        aug = iaa.Sequential([self.augmenters[aug_ind[0]],
+                              self.augmenters[aug_ind[1]],
+                              self.augmenters[aug_ind[2]]]
+                             )
+        return aug
+    
+    def generateStructuralDefect(self,image):
+        aug=self.randAugmenter()
+        image_array=(image.permute(1,2,0).numpy()*255).astype(np.uint8)# # *
+
+
+        image_array=aug(image=image_array)
+
+        height, width, _ = image_array.shape
+        grid_size = 8
+        cell_height = height // grid_size
+        cell_width = width // grid_size
+        
+        grid = []
+        for i in range(grid_size):
+            for j in range(grid_size):
+                cell = image_array[i * cell_height: (i + 1) * cell_height,
+                                j * cell_width: (j + 1) * cell_width, :]
+                grid.append(cell)
+        
+        np.random.shuffle(grid)
+        
+        reconstructed_image = np.zeros_like(image_array)
+        for i in range(grid_size):
+            for j in range(grid_size):
+                reconstructed_image[i * cell_height: (i + 1) * cell_height,
+                                    j * cell_width: (j + 1) * cell_width, :] = grid[i * grid_size + j]
+        return reconstructed_image
+
+
+
+class DefectGenerator():
+
+    def __init__(self, resize_shape=None,dtd_path="../../datasets/dtd/images"): 
+
+
+        self.texturalAnomalyGenerator=TexturalAnomalyGenerator(resize_shape,dtd_path)
+        self.structuralAnomalyGenerator=StructuralAnomalyGenerator(resize_shape)
+        
+        self.resize_shape=resize_shape
+        self.rot = iaa.Sequential([iaa.Affine(rotate=(-90, 90))])
+        self.toTensor=T.ToTensor()
+        
+    def generateMask(self,bMask):
+        perlin_scale = 6
+        min_perlin_scale = 0
+        perlin_scalex = 2 ** (torch.randint(min_perlin_scale, perlin_scale, (1,)).numpy()[0])
+        perlin_scaley = 2 ** (torch.randint(min_perlin_scale, perlin_scale, (1,)).numpy()[0])
+        perlin_noise = rand_perlin_2d_np((self.resize_shape[0], self.resize_shape[1]), (perlin_scalex, perlin_scaley))
+        perlin_noise = self.rot(image=perlin_noise)
+        threshold = 0.5
+        perlin_thr = np.where(perlin_noise > threshold, np.ones_like(perlin_noise), np.zeros_like(perlin_noise))
+        perlin_thr = np.expand_dims(perlin_thr, axis=2)
+        msk = (perlin_thr).astype(np.float32) 
+        msk=torch.from_numpy(msk).permute(2,0,1)
+        if (len(bMask)>0):
+            msk=bMask*msk
+        return msk
+    
+    def generateTexturalDefect(self, image,bMask=[]):
+        msk=torch.zeros((self.resize_shape[0], self.resize_shape[1]))
+        while (torch.count_nonzero(msk)<100):
+            msk=self.generateMask(bMask)*255.0
+        texturalImg=self.texturalAnomalyGenerator.getDtdImage()
+        texturalImg=torch.from_numpy(texturalImg).permute(2,0,1)/255.0
+        mskDtd=texturalImg*(msk)
+        
+        image = image * (1 - msk)+  (mskDtd) 
+        return image ,msk
+    
+    def generateStructuralDefect(self, image,bMask=[]):
+        msk=torch.zeros((self.resize_shape[0], self.resize_shape[1]))
+        while (torch.count_nonzero(msk)<100):
+            msk=self.generateMask(bMask)*255.0
+        structuralImg=self.structuralAnomalyGenerator.generateStructuralDefect(image)/255.0
+        structuralImg=torch.from_numpy(structuralImg).permute(2,0,1)
+        
+        mskDtd=structuralImg*(msk)
+        image = image * (1 - msk)+  (mskDtd) 
+        return image ,msk
+    
+    
+    def generateBlurredDefectiveImage(self, image,bMask=[]):
+        msk=torch.zeros((self.resize_shape[0], self.resize_shape[1]))
+        while (torch.count_nonzero(msk)<100):
+            msk=self.generateMask(bMask)*255.0
+        randGaussianValue = random.randint(0, 5)*2+21 
+        transform = T.GaussianBlur(kernel_size=(randGaussianValue, randGaussianValue), sigma=11.0)
+        imageBlurred = transform(image)
+        imageBlurred=imageBlurred*(msk)
+        image=image*(1-msk)
+
+        image=image+imageBlurred
+
+        return image,msk
+
+    def generateNsaDefect(self, image,bMask):
+        image = np.expand_dims(np.array(image),2) if len(np.array(image).shape)==2 else np.array(image)
+        image,msk=patch_ex(image,backgroundMask=bMask)
+        transform=T.ToTensor()
+        image = transform(image)
+        msk = transform(msk)*255.0
+        return image,msk
+    
+    
+    
+    def genSingleDefect(self,image,label,mskbg):
+        if label.lower() not in ["textural","structural","blurred","nsa"]:
+            raise ValueError("The defect type should be in ['textural','structural','blurred','nsa']")
+         
+        if (label.lower()=="textural" or label.lower()=="structural" or label.lower()=="blurred"):
+            imageT=self.toTensor(image)
+            bmask=self.toTensor(mskbg)
+            if (label.lower()=="textural"):
+                return self.generateTexturalDefect(imageT,bmask)
+            elif (label.lower()=="structural"):
+                return self.generateStructuralDefect(imageT,bmask)
+            elif (label.lower()=="blurred"):
+                return self.generateBlurredDefectiveImage(imageT,bmask)
+        elif (label.lower()=="nsa"):
+            return self.generateNsaDefect(image,mskbg) 
+
+    def genDefect(self,image,defectType,category="",return_list=False):
+        mskbg=backGroundMask(image,obj=category) 
+        if not return_list:
+            if (len(defectType)>1):
+                index=np.random.randint(0,len(defectType))
+                label=defectType[index]
+            else:
+                label=defectType[0]
+            return self.genSingleDefect(image,label,mskbg)            
+        if return_list: 
+            defectImages=[]
+            defectMasks=[]
+            for label in defectType:
+                defectImage,defectMask=self.genSingleDefect(image,label,mskbg)
+                defectImages.append(defectImage)
+                defectMasks.append(defectMask)
+            return defectImages,defectMasks

source/nsa.py

@@ -0,0 +1,300 @@
+import numpy as np
+import cv2
+import sys
+from skimage.morphology import disk
+from skimage.filters import median
+import torch
+import torchvision.transforms as T
+import random
+
+import matplotlib.pyplot as plt
+from PIL import Image
+
+BACKGROUND = {'bottle':(200, 60), 'screw':(200, 60), 'capsule':(200, 60), 'zipper':(200, 60), 
+              'hazelnut':(20, 20), 'pill':(20, 20), 'toothbrush':(20, 20), 'metal_nut':(20, 20)}
+
+
+def backGroundMask(image,obj=""):
+    image = np.expand_dims(np.array(image),2) if len(np.array(image).shape)==2 else np.array(image)
+    #if obj=="":
+    if obj not in BACKGROUND.keys():
+        return np.ones_like(image[...,0:1])
+    else:
+        skip_background=BACKGROUND[obj]
+    if isinstance(skip_background, tuple):
+        skip_background = [skip_background]
+    object_mask = np.ones_like(image[...,0:1])
+    for background, threshold in skip_background:
+        object_mask &= np.uint8(np.abs(image.mean(axis=-1, keepdims=True) - background) > threshold)
+    object_mask[...,0] = cv2.medianBlur(object_mask[...,0], 7)  # remove grain from threshold choice
+
+    return object_mask
+
+
+def patch_ex(ima_dest, ima_src=None, same=False, num_patches=1,
+             mode=cv2.NORMAL_CLONE, width_bounds_pct=((0.05,0.2),(0.05,0.2)), min_object_pct=0.25, 
+             min_overlap_pct=0.25, shift=True, label_mode='binary', backgroundMask=None, tol=1, resize=True,
+             gamma_params=None, intensity_logistic_params=(1/6, 20),
+             resize_bounds=(0.7, 1.3), num_ellipses=None, verbose=True, cutpaste_patch_generation=False):
+    """
+    Create a synthetic training example from the given images by pasting/blending random patches.
+    Args:
+        ima_dest (uint8 numpy array): image with shape (W,H,3) or (W,H,1) where patch should be changed
+        ima_src (uint8 numpy array): optional, otherwise use ima_dest as source
+        same (bool): use ima_dest as source even if ima_src given
+        mode: 'uniform', 'swap', 'mix', cv2.NORMAL_CLONE, or cv2.MIXED_CLONE what blending method to use
+             ('mix' is flip a coin between normal and mixed clone)
+        num_patches (int): how many patches to add. the method will always attempt to add the first patch,
+                    for each subsequent patch it flips a coin
+        width_bounds_pct ((float, float), (float, float)): min half-width of patch ((min_dim1, max_dim1), (min_dim2, max_dim2))
+        shift (bool): if false, patches in src and dest image have same coords. otherwise random shift
+        resize (bool): if true, patch is resampled at random size (within bounds and keeping aspect ratio the same) before blending  
+        skip_background (int, int) or [(int, int),]: optional, assume background color is first and only interpolate patches
+                    in areas where dest or src patch has pixelwise MAD < second from background.
+        tol (int): mean abs intensity change required to get positive label
+        gamma_params (float, float, float): optional, (shape, scale, left offset) of gamma dist to sample half-width of patch from,
+                    otherwise use uniform dist between 0.05 and 0.95
+        intensity_logistic_params (float, float): k, x0 of logitistc map for intensity based label
+        num_ellipses (int): optional, if set, the rectangular patch mask is filled with random ellipses
+        label_mode: 'binary', 
+                    'continuous' -- use interpolation factor as label (only when mode is 'uniform'),
+                    'intensity' -- use median filtered mean absolute pixelwise intensity difference as label,
+                    'logistic-intensity' -- use logistic median filtered of mean absolute pixelwise intensity difference as label,
+        cutpaste_patch_generation (bool): optional, if set, width_bounds_pct, resize, skip_background, min_overlap_pct, min_object_pct, 
+                    num_patches and gamma_params are ignored. A single patch is sampled as in the CutPaste paper: 
+                        1. sampling the area ratio between the patch and the full image from (0.02, 0.15)
+                        2. determine the aspect ratio by sampling from (0.3, 1) union (1, 3.3)
+                        3. sample location such that patch is contained entirely within the image
+    """
+    if mode == 'mix':
+        mode = (cv2.NORMAL_CLONE, cv2.MIXED_CLONE)[np.random.randint(2)]
+
+    if cutpaste_patch_generation:
+        width_bounds_pct = None 
+        resize = False 
+        min_overlap_pct = None 
+        min_object_pct = None
+        gamma_params = None
+        num_patches = 1
+
+    ima_src = ima_dest.copy() if same or (ima_src is None) else ima_src
+
+    src_object_mask = backgroundMask
+    dest_object_mask = backgroundMask
+    
+    
+    mask = np.zeros_like(ima_dest[..., 0:1]) 
+    patchex = ima_dest.copy()
+    coor_min_dim1, coor_max_dim1, coor_min_dim2, coor_max_dim2 = mask.shape[0] - 1, 0, mask.shape[1] - 1, 0 
+    if label_mode == 'continuous':
+        factor = np.random.uniform(0.05, 0.95)
+    else:
+        factor = 1
+    for i in range(num_patches):
+        if i == 0 or np.random.randint(2) > 0: 
+            patchex, ((_coor_min_dim1, _coor_max_dim1), (_coor_min_dim2, _coor_max_dim2)), patch_mask = _patch_ex(
+                patchex, ima_src, dest_object_mask, src_object_mask, mode, label_mode, shift, resize, width_bounds_pct, 
+                gamma_params, min_object_pct, min_overlap_pct, factor, resize_bounds, num_ellipses, verbose, cutpaste_patch_generation)
+            if patch_mask is not None:
+                mask[_coor_min_dim1:_coor_max_dim1,_coor_min_dim2:_coor_max_dim2] = patch_mask
+                coor_min_dim1 = min(coor_min_dim1, _coor_min_dim1) 
+                coor_max_dim1 = max(coor_max_dim1, _coor_max_dim1) 
+                coor_min_dim2 = min(coor_min_dim2, _coor_min_dim2)
+                coor_max_dim2 = max(coor_max_dim2, _coor_max_dim2)
+
+    # create label
+    label_mask = np.uint8(np.mean(np.abs(1.0 * mask*ima_dest - 1.0 * mask*patchex), axis=-1, keepdims=True) > tol)
+    label_mask[...,0] = cv2.medianBlur(label_mask[...,0], 5) 
+
+    if label_mode == 'continuous':
+        label = label_mask * factor
+    elif label_mode in ['logistic-intensity', 'intensity']:
+        k, x0 = intensity_logistic_params
+        label = np.mean(np.abs(label_mask * ima_dest * 1.0 - label_mask * patchex * 1.0), axis=-1, keepdims=True)
+        label[...,0] = median(label[...,0], disk(5))
+        if label_mode == 'logistic-intensity':
+            label = label_mask / (1 + np.exp(-k * (label - x0)))
+    elif label_mode == 'binary':
+        label = label_mask  
+    else:
+        raise ValueError("label_mode not supported" + str(label_mode))
+
+    return patchex, label
+
+
+def _patch_ex(ima_dest, ima_src, dest_object_mask, src_object_mask, mode, label_mode, shift, resize, width_bounds_pct, 
+              gamma_params, min_object_pct, min_overlap_pct, factor, resize_bounds, num_ellipses, verbose, cutpaste_patch_generation):
+    if cutpaste_patch_generation:
+        skip_background = False
+        dims = np.array(ima_dest.shape)
+        if dims[0] != dims[1]:
+            raise ValueError("CutPaste patch generation only works for square images")
+        # 1. sampling the area ratio between the patch and the full image from (0.02, 0.15)
+        # (divide by 4 as patch-widths below are actually half-widths)
+        area_ratio = np.random.uniform(0.02, 0.15) / 4.0
+        #  2. determine the aspect ratio by sampling from (0.3, 1) union (1, 3.3)
+        if np.random.randint(2) > 0:
+            aspect_ratio = np.random.uniform(0.3, 1)
+        else:
+            aspect_ratio = np.random.uniform(1, 3.3)
+        
+        patch_width_dim1 = int(np.rint(np.clip(np.sqrt(area_ratio * aspect_ratio * dims[0]**2), 0, dims[0])))
+        patch_width_dim2 = int(np.rint(np.clip(area_ratio * dims[0]**2 / patch_width_dim1, 0, dims[1])))
+        #  3. sample location such that patch is contained entirely within the image
+        center_dim1 = np.random.randint(patch_width_dim1, dims[0] - patch_width_dim1)
+        center_dim2 = np.random.randint(patch_width_dim2, dims[1] - patch_width_dim2)
+
+        coor_min_dim1 = np.clip(center_dim1 - patch_width_dim1, 0, dims[0])
+        coor_min_dim2 = np.clip(center_dim2 - patch_width_dim2, 0, dims[1])
+        coor_max_dim1 = np.clip(center_dim1 + patch_width_dim1, 0, dims[0])
+        coor_max_dim2 = np.clip(center_dim2 + patch_width_dim2, 0, dims[1])
+
+        patch_mask = np.ones((coor_max_dim1 - coor_min_dim1, coor_max_dim2 - coor_min_dim2, 1), dtype=np.uint8)
+    else:
+        skip_background = (src_object_mask is not None) and (dest_object_mask is not None)
+        dims = np.array(ima_dest.shape)
+        min_width_dim1 = (width_bounds_pct[0][0]*dims[0]).round().astype(int)
+        max_width_dim1 = (width_bounds_pct[0][1]*dims[0]).round().astype(int)
+        min_width_dim2 = (width_bounds_pct[1][0]*dims[1]).round().astype(int)
+        max_width_dim2 = (width_bounds_pct[1][1]*dims[1]).round().astype(int)
+
+        if gamma_params is not None:
+            shape, scale, lower_bound = gamma_params
+            patch_width_dim1 = int(np.clip((lower_bound + np.random.gamma(shape, scale)) * dims[0], min_width_dim1, max_width_dim1))
+            patch_width_dim2 = int(np.clip((lower_bound + np.random.gamma(shape, scale)) * dims[1], min_width_dim2, max_width_dim2))
+        else:
+            patch_width_dim1 = np.random.randint(min_width_dim1, max_width_dim1)
+            patch_width_dim2 = np.random.randint(min_width_dim2, max_width_dim2)
+
+        found_patch = False
+        attempts = 0
+        while not found_patch:
+            center_dim1 = np.random.randint(min_width_dim1, dims[0]-min_width_dim1)
+            center_dim2 = np.random.randint(min_width_dim2, dims[1]-min_width_dim2)
+
+            coor_min_dim1 = np.clip(center_dim1 - patch_width_dim1, 0, dims[0])
+            coor_min_dim2 = np.clip(center_dim2 - patch_width_dim2, 0, dims[1])
+            coor_max_dim1 = np.clip(center_dim1 + patch_width_dim1, 0, dims[0])
+            coor_max_dim2 = np.clip(center_dim2 + patch_width_dim2, 0, dims[1])
+
+            if num_ellipses is not None:
+                ellipse_min_dim1 = min_width_dim1
+                ellipse_min_dim2 = min_width_dim2
+                ellipse_max_dim1 = max(min_width_dim1 + 1, patch_width_dim1 // 2)
+                ellipse_max_dim2 = max(min_width_dim2 + 1, patch_width_dim2 // 2)
+                patch_mask = np.zeros((coor_max_dim1 - coor_min_dim1, coor_max_dim2 - coor_min_dim2), dtype=np.uint8) 
+                x = np.arange(patch_mask.shape[0]).reshape(-1, 1)
+                y = np.arange(patch_mask.shape[1]).reshape(1, -1)
+                for _ in range(num_ellipses):
+                    theta = np.random.uniform(0, np.pi)
+                    x0 = np.random.randint(0, patch_mask.shape[0])
+                    y0 = np.random.randint(0, patch_mask.shape[1])
+                    a = np.random.randint(ellipse_min_dim1, ellipse_max_dim1)
+                    b = np.random.randint(ellipse_min_dim2, ellipse_max_dim2)
+                    ellipse = (((x-x0)*np.cos(theta) + (y-y0)*np.sin(theta))/a)**2 + (((x-x0)*np.sin(theta) + (y-y0)*np.cos(theta))/b)**2 <= 1  # True for points inside the ellipse
+                    patch_mask |= ellipse
+                patch_mask = patch_mask[...,None]
+            else:
+                patch_mask = np.ones((coor_max_dim1 - coor_min_dim1, coor_max_dim2 - coor_min_dim2, 1), dtype=np.uint8) 
+
+            if skip_background:
+                background_area = np.sum(patch_mask & src_object_mask[coor_min_dim1:coor_max_dim1, coor_min_dim2:coor_max_dim2])
+                if num_ellipses is not None:
+                    patch_area = np.sum(patch_mask)
+                else:
+                    patch_area = patch_mask.shape[0] * patch_mask.shape[1]
+                found_patch = (background_area / patch_area > min_object_pct) 
+            else:
+                found_patch = True
+            attempts += 1
+            if attempts == 200:
+                if verbose:
+                    print('No suitable patch found.')
+                return ima_dest.copy(), ((0,0),(0,0)), None
+    src = ima_src[coor_min_dim1:coor_max_dim1, coor_min_dim2:coor_max_dim2]
+    height, width, _ = src.shape
+    if resize:
+        lb, ub = resize_bounds
+        scale = np.clip(np.random.normal(1, 0.5), lb, ub)
+        new_height = np.clip(scale * height, min_width_dim1, max_width_dim1)
+        new_width = np.clip(int(new_height / height * width), min_width_dim2, max_width_dim2)
+        new_height = np.clip(int(new_width / width * height), min_width_dim1, max_width_dim1)  # in case there was clipping
+        
+        if src.shape[2] == 1:  # grayscale
+            src = cv2.resize(src[..., 0], (new_width, new_height))
+            src = src[...,None]
+        else:
+            src = cv2.resize(src, (new_width, new_height))
+        height, width, _ = src.shape
+        patch_mask = cv2.resize(patch_mask[...,0], (width, height))
+        patch_mask = patch_mask[...,None]
+
+    if skip_background:
+        src_object_mask = cv2.resize(src_object_mask[coor_min_dim1:coor_max_dim1, coor_min_dim2:coor_max_dim2, 0], (width, height))
+        src_object_mask = src_object_mask[...,None]
+    
+    # sample destination location and size
+    if shift:
+        found_center = False
+        attempts = 0
+        while not found_center:
+            center_dim1 = np.random.randint(height//2 + 1, ima_dest.shape[0] - height//2 - 1)
+            center_dim2 = np.random.randint(width//2 + 1, ima_dest.shape[1] - width//2 - 1)
+            coor_min_dim1, coor_max_dim1 = center_dim1 - height//2, center_dim1 + (height+1)//2
+            coor_min_dim2, coor_max_dim2 = center_dim2 - width//2, center_dim2 + (width+1)//2
+
+            if skip_background: 
+                src_and_dest = dest_object_mask[coor_min_dim1:coor_max_dim1, coor_min_dim2:coor_max_dim2] & src_object_mask & patch_mask
+                src_or_dest = (dest_object_mask[coor_min_dim1:coor_max_dim1, coor_min_dim2:coor_max_dim2] | src_object_mask) & patch_mask
+                found_center = (np.sum(src_object_mask) / (patch_mask.shape[0] * patch_mask.shape[1]) > min_object_pct and    # contains object
+                            np.sum(src_and_dest) / np.sum(src_object_mask) > min_overlap_pct)                    # object overlaps src object
+            else:
+                found_center = True
+            attempts += 1
+            if attempts == 200:
+                if verbose:
+                    print('No suitable center found. Dims were:', width, height)
+                return ima_dest.copy(), ((0,0),(0,0)), None
+            
+    # blend
+    if skip_background:
+        patch_mask &= src_object_mask | dest_object_mask[coor_min_dim1:coor_max_dim1, coor_min_dim2:coor_max_dim2] 
+
+    if mode == 'swap':
+        patchex = ima_dest.copy()
+        before = patchex[coor_min_dim1:coor_max_dim1, coor_min_dim2:coor_max_dim2]
+        patchex[coor_min_dim1:coor_max_dim1, coor_min_dim2:coor_max_dim2] -= patch_mask * before
+        patchex[coor_min_dim1:coor_max_dim1, coor_min_dim2:coor_max_dim2] += patch_mask * src
+    elif mode == 'uniform':
+        patchex = 1.0 * ima_dest
+        before = patchex[coor_min_dim1:coor_max_dim1, coor_min_dim2:coor_max_dim2]
+        patchex[coor_min_dim1:coor_max_dim1, coor_min_dim2:coor_max_dim2] -= factor * patch_mask * before
+        patchex[coor_min_dim1:coor_max_dim1, coor_min_dim2:coor_max_dim2] += factor * patch_mask * src
+        patchex = np.uint8(np.floor(patchex))
+    elif mode in [cv2.NORMAL_CLONE, cv2.MIXED_CLONE]:  # poisson interpolation
+        int_factor = np.uint8(np.ceil(factor * 255))
+        # add background to patchmask to avoid artefacts
+        if skip_background:
+            patch_mask_scaled = int_factor * (patch_mask | ((1 - src_object_mask) & (1 - dest_object_mask[coor_min_dim1:coor_max_dim1, coor_min_dim2:coor_max_dim2])))
+        else:
+            patch_mask_scaled = int_factor * patch_mask
+        patch_mask_scaled[0], patch_mask_scaled[-1], patch_mask_scaled[:,0], patch_mask_scaled[:,-1] = 0, 0, 0, 0  # zero border to avoid artefacts
+        center = (coor_max_dim2 - (coor_max_dim2 - coor_min_dim2) // 2, coor_min_dim1 + (coor_max_dim1 - coor_min_dim1) // 2)  # height dim first
+        if np.sum(patch_mask_scaled > 0) < 50:  # cv2 seamlessClone will fail if positive mask area is too small
+            return ima_dest.copy(), ((0,0),(0,0)), None
+        try:
+            if ima_dest.shape[2] == 1:  # grayscale
+                # pad to 3 channels as that's what OpenCV expects
+                src_3 = np.concatenate((src, np.zeros_like(src), np.zeros_like(src)), axis=2)
+                ima_dest_3 = np.concatenate((ima_dest, np.zeros_like(ima_dest), np.zeros_like(ima_dest)), axis=2)
+                patchex = cv2.seamlessClone(src_3, ima_dest_3, patch_mask_scaled, center, mode)
+                patchex = patchex[...,0:1]  # extract first channel
+            else:  # RGB
+                patchex = cv2.seamlessClone(src, ima_dest, patch_mask_scaled, center, mode)
+        except cv2.error as e:
+            print('WARNING, tried bad interpolation mask and got:', e)
+            return ima_dest.copy(), ((0,0),(0,0)), None
+    else:
+        raise ValueError("mode not supported" + str(mode))
+
+    return patchex, ((coor_min_dim1, coor_max_dim1), (coor_min_dim2, coor_max_dim2)), patch_mask

source/perlin.py

@@ -0,0 +1,99 @@
+import torch
+import math
+import numpy as np
+
+def lerp_np(x,y,w):
+    fin_out = (y-x)*w + x
+    return fin_out
+
+def generate_fractal_noise_2d(shape, res, octaves=1, persistence=0.5):
+    noise = np.zeros(shape)
+    frequency = 1
+    amplitude = 1
+    for _ in range(octaves):
+        noise += amplitude * generate_perlin_noise_2d(shape, (frequency*res[0], frequency*res[1]))
+        frequency *= 2
+        amplitude *= persistence
+    return noise
+
+
+def generate_perlin_noise_2d(shape, res):
+    def f(t):
+        return 6 * t ** 5 - 15 * t ** 4 + 10 * t ** 3
+
+    delta = (res[0] / shape[0], res[1] / shape[1])
+    d = (shape[0] // res[0], shape[1] // res[1])
+    grid = np.mgrid[0:res[0]:delta[0], 0:res[1]:delta[1]].transpose(1, 2, 0) % 1
+    # Gradients
+    angles = 2 * np.pi * np.random.rand(res[0] + 1, res[1] + 1)
+    gradients = np.dstack((np.cos(angles), np.sin(angles)))
+    g00 = gradients[0:-1, 0:-1].repeat(d[0], 0).repeat(d[1], 1)
+    g10 = gradients[1:, 0:-1].repeat(d[0], 0).repeat(d[1], 1)
+    g01 = gradients[0:-1, 1:].repeat(d[0], 0).repeat(d[1], 1)
+    g11 = gradients[1:, 1:].repeat(d[0], 0).repeat(d[1], 1)
+    # Ramps
+    n00 = np.sum(grid * g00, 2)
+    n10 = np.sum(np.dstack((grid[:, :, 0] - 1, grid[:, :, 1])) * g10, 2)
+    n01 = np.sum(np.dstack((grid[:, :, 0], grid[:, :, 1] - 1)) * g01, 2)
+    n11 = np.sum(np.dstack((grid[:, :, 0] - 1, grid[:, :, 1] - 1)) * g11, 2)
+    # Interpolation
+    t = f(grid)
+    n0 = n00 * (1 - t[:, :, 0]) + t[:, :, 0] * n10
+    n1 = n01 * (1 - t[:, :, 0]) + t[:, :, 0] * n11
+    return np.sqrt(2) * ((1 - t[:, :, 1]) * n0 + t[:, :, 1] * n1)
+
+
+def rand_perlin_2d_np(shape, res, fade=lambda t: 6 * t ** 5 - 15 * t ** 4 + 10 * t ** 3):
+    delta = (res[0] / shape[0], res[1] / shape[1])
+    d = (shape[0] // res[0], shape[1] // res[1])
+    grid = np.mgrid[0:res[0]:delta[0], 0:res[1]:delta[1]].transpose(1, 2, 0) % 1
+
+    angles = 2 * math.pi * np.random.rand(res[0] + 1, res[1] + 1)
+    gradients = np.stack((np.cos(angles), np.sin(angles)), axis=-1)
+    tt = np.repeat(np.repeat(gradients,d[0],axis=0),d[1],axis=1)
+
+    tile_grads = lambda slice1, slice2: np.repeat(np.repeat(gradients[slice1[0]:slice1[1], slice2[0]:slice2[1]],d[0],axis=0),d[1],axis=1)
+    dot = lambda grad, shift: (
+                np.stack((grid[:shape[0], :shape[1], 0] + shift[0], grid[:shape[0], :shape[1], 1] + shift[1]),
+                            axis=-1) * grad[:shape[0], :shape[1]]).sum(axis=-1)
+
+    n00 = dot(tile_grads([0, -1], [0, -1]), [0, 0])
+    n10 = dot(tile_grads([1, None], [0, -1]), [-1, 0])
+    n01 = dot(tile_grads([0, -1], [1, None]), [0, -1])
+    n11 = dot(tile_grads([1, None], [1, None]), [-1, -1])
+    t = fade(grid[:shape[0], :shape[1]])
+    return math.sqrt(2) * lerp_np(lerp_np(n00, n10, t[..., 0]), lerp_np(n01, n11, t[..., 0]), t[..., 1])
+
+
+def rand_perlin_2d(shape, res, fade=lambda t: 6 * t ** 5 - 15 * t ** 4 + 10 * t ** 3):
+    delta = (res[0] / shape[0], res[1] / shape[1])
+    d = (shape[0] // res[0], shape[1] // res[1])
+
+    grid = torch.stack(torch.meshgrid(torch.arange(0, res[0], delta[0]), torch.arange(0, res[1], delta[1])), dim=-1) % 1
+    angles = 2 * math.pi * torch.rand(res[0] + 1, res[1] + 1)
+    gradients = torch.stack((torch.cos(angles), torch.sin(angles)), dim=-1)
+
+    tile_grads = lambda slice1, slice2: gradients[slice1[0]:slice1[1], slice2[0]:slice2[1]].repeat_interleave(d[0],
+                                                                                                              0).repeat_interleave(
+        d[1], 1)
+    dot = lambda grad, shift: (
+                torch.stack((grid[:shape[0], :shape[1], 0] + shift[0], grid[:shape[0], :shape[1], 1] + shift[1]),
+                            dim=-1) * grad[:shape[0], :shape[1]]).sum(dim=-1)
+    n00 = dot(tile_grads([0, -1], [0, -1]), [0, 0])
+
+    n10 = dot(tile_grads([1, None], [0, -1]), [-1, 0])
+    n01 = dot(tile_grads([0, -1], [1, None]), [0, -1])
+    n11 = dot(tile_grads([1, None], [1, None]), [-1, -1])
+    t = fade(grid[:shape[0], :shape[1]])
+    return math.sqrt(2) * torch.lerp(torch.lerp(n00, n10, t[..., 0]), torch.lerp(n01, n11, t[..., 0]), t[..., 1])
+
+
+def rand_perlin_2d_octaves(shape, res, octaves=1, persistence=0.5):
+    noise = torch.zeros(shape)
+    frequency = 1
+    amplitude = 1
+    for _ in range(octaves):
+        noise += amplitude * rand_perlin_2d(shape, (frequency * res[0], frequency * res[1]))
+        frequency *= 2
+        amplitude *= persistence
+    return noise