Captcha - localo

Category: Misc
Difficulty: Medium/Hard
Author: LiveOverflow

Description

"The Enrichment Center regrets to inform you that this next test is impossible. Make no attempt to solve it. No one will blame you for giving up. In fact, quitting at this point is a perfectly reasonable response."

http://hax1.allesctf.net:9200/

Summery

The author provided a link to a website, we need to solve multiple Captcha stages to get the flag. If we fail on any Captcha, we have to try again from the beginning.

Solution

If we fail at the first stage, we get the solution as the response. We can use that information to automatically generate and label a set to train an AI. The problem is not new and I took a look at many approaches since I have never done any machine learning (I just tried random adjustments to my AI until it kind of worked..). I used Keras link since it is beginner friendly and simple. I wrote an algorithm to separate the image into smaller pieces containing just one letter, but the algorithm didn't catch some cases, especially if there are two characters that are very close together and overlap. Therefore I used the data produced by the algorithm to train another AI. I manually sorted false labels. That AI is used to preprocess the segmented images, if the AI "thinks" that there is more than one character on the image, it splits the image in the middle. I could have used the aspect ratio, to guess if there are more than two overlapping characters, but since this is quite rare I didn't care. After training and readjusting the AI for a while, and gathering more data for problem characters, it got the flag after 40 tries. The lowercase i was mean.

Code

Sorry for my dirty code 😛

code for solving:

from keras.models import load_model
import numpy as np
import cv2
import pickle
import segmentation.predict as predict
with open("/app/model/labels.dat", "rb") as f:
    lb = pickle.load(f)

model = load_model("/app/model/model.hdf5")

def solve(name):
    image = cv2.imread(name)
    gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
    gray = cv2.copyMakeBorder(gray, 8, 8, 8, 8, cv2.BORDER_CONSTANT,value=(255,255,255))
    gray[gray > 60] = 255

    #image = cv2.copyMakeBorder(image, 8, 8, 8, 8, cv2.BORDER_CONSTANT,value=(255,255,255))
    thresh = cv2.threshold(gray, 0, 255, cv2.THRESH_BINARY_INV | cv2.THRESH_OTSU)[1]

    cnts = cv2.findContours(thresh.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)[0]
    
    regions = {}

    tolerance = 5
    last = -1

    for cnt in cnts:
        (x, y, w, h) = cv2.boundingRect(cnt)
        if (w*h <= 3**2 and w/h == 1) or w*h < 7:
            continue
        if x in regions:
            (x1, y1, w1, h1) = regions[x]
            x_ = min(x,x1)
            y_ = min(y,y1)
            ww_ = max(x+w,x1+w1)
            hh_ = max(y+h,y1+h1)
            regions[x]=(x_,y_,ww_-x_,hh_-y_)
            continue
        regions[x]=(x,y,w,h)
    for k in sorted(regions):
        (x, y, w, h) = regions[k]
        if last >= 0:
            (x1, y1, w1, h1) = regions[last]
            if ((abs(1-w/h) < 0.35 and w <8 and h <8) or (abs(1-w1/h1) < 0.35) and w1 <8 and h1 <8):
                if x < x1 + w1 +tolerance and  x + w +tolerance > x1:
                    x_ = min(x,x1)
                    y_ = min(y,y1)
                    ww_ = max(x+w,x1+w1)
                    hh_ = max(y+h,y1+h1)
                    regions[last]=(x_,y_,ww_-x_,hh_-y_)
                    del regions[k]
                    #cv2.rectangle(image, (x_,y_), (ww_,hh_), (255, 255, 0), 1)
                    continue
            elif x < x1 + w1 and x + w > x1:
                x_ = min(x,x1)
                y_ = min(y,y1)
                ww_ = max(x+w,x1+w1)
                hh_ = max(y+h,y1+h1)
                regions[last]=(x_,y_,ww_-x_,hh_-y_)
                del regions[k]
                #cv2.rectangle(image, (x_,y_), (ww_,hh_), (255, 255, 0), 1)
                continue

        regions[x]=(x, y, w, h)
        last = x
            #cv2.rectangle(image, (x,y), (w,h), (255, 0, 0), 1)

    for k in sorted(regions):
        (x, y, w, h) = regions[k]
        letter = gray[y-2:y + h+2, x-2 :x + w +2 ]
        n = predict.is_two(letter)
        if n == 2:
            hw = w // 2
            regions[x]=(x, y, hw, h)
            regions[x+hw]=(x + hw, y, hw, h)

    ret = []
    for k in sorted(regions):
        x, y, w, h = regions[k]

        letter = gray[y-2:y + h+2, x-2 :x + w +2 ]
        letter = cv2.resize(letter, (20, 20))
        letter = np.expand_dims(letter, axis=2)
        letter = np.expand_dims(letter, axis=0)

        prediction = model.predict(letter)

        c = lb.inverse_transform(prediction)[0]
        ret.append(c)
    return ''.join(ret)

import requests
import os
import time
import cv2
import base64
from PIL import Image
import solve
import random
import string
from io import BytesIO
import time
import json

s = requests.Session()
url = "http://hax1.allesctf.net:9200"
base = "/app"


def do(data,stage):
    st = time.perf_counter()
    im = base64.decodebytes(data.encode())
    try:
        os.makedirs(base+("/temp/%d"%stage))
    except:
        pass
    name = base+"/temp/%d/%s.bmp" %(stage,''.join(random.choices(string.ascii_lowercase,k=10)))
    with open(name, "wb") as f:
        f.write(im)
    sol = solve.solve(name)
    #print((time.perf_counter()-st))
    return sol

class Stage:
    def __init__(self,s,url):
        self.url = url
        self.s =s

    def run(self):
        pass

class StageN(Stage):

    def run(self,stage):
        r = s.get(url+"/captcha/%d.html" %(stage))
        if stage == 4:
            print(r.text)
        pics = r.text.split('<form action="" method="post">')[1].split('<img src="')
        payload = {}
        for idx,p in enumerate(pics):
            if idx == 0:
                continue
            payload[idx-1]=do(p.split('">')[0].split("base64,")[1],stage)
        if stage==3:
            wd = r.text
            for idx, p in payload.items():
                wd = wd.replace('<input type="text" name="%d">' %idx, '<input type="text" name="%d" value="%s">'%(idx,p))
            #with open("/app/data.json","w") as f:
            #    f.write(json.dumps(payload))
            with open("/app/data.html","w") as f:
                f.write(wd)
            
            #screenshot(s,url+"/captcha/%d.html" %(stage))
        r = s.post(url+"/captcha/%d.html" %(stage),data=payload)
        
        if "Human detected" in r.text:
            print("failed stage %d" %stage)
            #print(payload)
            #print(r.text)
            #exit(1)
            return False
        return StageN(s,url).run(stage+1)

class Stage0(Stage):

    def run(self):
        r = s.get(url+"/captcha/0")
        imb64 = r.text.split('<form action="" method="post">')[1].split('<img src="')[1].split('">')[0].split("base64,")[1]

        solution = do(imb64,0)
        r = s.post(url+"/captcha/0",data={
            '0':solution
        })
        text = r.text
        if not "Since you are thankfully not humans" in text or not StageN(s,url).run(1):
            if "The solution would have been" in text:
                #print(r.text)
                real =text.split('The solution would have been <b>')[1].split('</b>')[0]
                print("---------\n%s\n%s\n---------" % (solution,real))
            Stage0(s,url).run()
        return True
Stage0(s,url).run()
driver.quit()

code for recognition data collection:

import requests
import os
import time
import cv2
import base64
from PIL import Image
import string
from threading import Thread, Lock
import sys
from queue import Queue
import time
import random
import segmentation.predict as predict

data = {}

def collect():
    s = requests.Session()
    url = "http://hax1.allesctf.net:9200"
    base="/app/train/temp"
    r = s.get(url+"/captcha/0")
    imb64 = r.text.split('<form action="" method="post">')[1].split('<img src="')[1].split('">')[0].split("base64,")[1]
    im = base64.decodebytes(imb64.encode())
    try:
        os.makedirs(base+"/failed")
    except:
        pass
    base_name = ''.join(random.choices(string.ascii_lowercase,k=5))
    name = base+"/%s.bmp" %(base_name)
    with open(name, "wb") as f:
        f.write(im)

    r = s.post(url+"/captcha/0",data={'0':''})
    solution =r.text.split('The solution would have been <b>')[1].split('</b>')[0]

    data[solution]=base_name

def sort_data(base_name,solution):
    base="/app/train/temp"
    name = base+"/%s.bmp" %(base_name)

    image = cv2.imread(name)
    gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
    gray = cv2.copyMakeBorder(gray, 8, 8, 8, 8, cv2.BORDER_CONSTANT,value=(255,255,255))
    gray[gray > 60] = 255

    #image = cv2.copyMakeBorder(image, 8, 8, 8, 8, cv2.BORDER_CONSTANT,value=(255,255,255))
    thresh = cv2.threshold(gray, 0, 255, cv2.THRESH_BINARY_INV | cv2.THRESH_OTSU)[1]

    cnts = cv2.findContours(thresh.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)[0]
    
    regions = {}

    tolerance = 5
    last = -1

    for cnt in cnts:
        (x, y, w, h) = cv2.boundingRect(cnt)
        if (w*h <= 3**2 and w/h == 1) or w*h < 7:
            continue
        if x in regions:
            (x1, y1, w1, h1) = regions[x]
            x_ = min(x,x1)
            y_ = min(y,y1)
            ww_ = max(x+w,x1+w1)
            hh_ = max(y+h,y1+h1)
            regions[x]=(x_,y_,ww_-x_,hh_-y_)
            continue
        regions[x]=(x,y,w,h)
    for k in sorted(regions):
        (x, y, w, h) = regions[k]
        if last >= 0:
            (x1, y1, w1, h1) = regions[last]
            if ((abs(1-w/h) < 0.35 and w <8 and h <8) or (abs(1-w1/h1) < 0.35) and w1 <8 and h1 <8):
                if x < x1 + w1 +tolerance and  x + w +tolerance > x1:
                    x_ = min(x,x1)
                    y_ = min(y,y1)
                    ww_ = max(x+w,x1+w1)
                    hh_ = max(y+h,y1+h1)
                    regions[last]=(x_,y_,ww_-x_,hh_-y_)
                    del regions[k]
                    #cv2.rectangle(image, (x_,y_), (ww_,hh_), (255, 255, 0), 1)
                    continue
            elif x < x1 + w1 and x + w > x1:
                x_ = min(x,x1)
                y_ = min(y,y1)
                ww_ = max(x+w,x1+w1)
                hh_ = max(y+h,y1+h1)
                regions[last]=(x_,y_,ww_-x_,hh_-y_)
                del regions[k]
                #cv2.rectangle(image, (x_,y_), (ww_,hh_), (255, 255, 0), 1)
                continue

        regions[x]=(x, y, w, h)
        last = x
            #cv2.rectangle(image, (x,y), (w,h), (255, 0, 0), 1)

    for k in sorted(regions):
        (x, y, w, h) = regions[k]
        letter = gray[y-2:y + h+2, x-2 :x + w +2 ]
        n = predict.is_two(letter)
        if n == 2:
            hw = w // 2
            regions[x]=(x, y, hw, h)
            regions[x+hw]=(x + hw, y, hw, h)
            last = x+hw

    #cv2.imwrite(name+".bmp", image)

    i = 0
    
    if len(solution) != len(regions.keys()):
        cv2.imwrite(base+"/failed/"+"%s.bmp" %(solution), image)
        print("FAIL")
        return

    for k in sorted(regions):
        x, y, w, h = regions[k]

        letter = gray[y-2:y + h+2, x-2 :x + w +2 ]

        
        name = "/app/train/%c/%s_%s.bmp" %(solution[i],base_name,''.join(random.choices(string.ascii_lowercase,k=2)))
        try:
            os.makedirs("/app/train/%c" %(solution[i]))
        except:
            pass
        i+=1

        cv2.imwrite(name, letter)
   
concurrent = 40
n = 1000
q = Queue(concurrent)
def worker():
    while True:    
        q.get()()
        q.task_done()
        time.sleep(0.1)
j = 0
def adder():
    for i in range(n):
        q.put(collect)        

for i in range(concurrent):
    t = Thread(target=worker)
    t.daemon = True
    t.start()
try:
    t2 = Thread(target = adder)
    t2.daemon = True
    t2.start()
    time.sleep(1)
    while q.qsize() > 0 or len(data) > 0:
        if  len(data) > 0:
            print("[%d/%d]" %(j,n))
            k,v = data.popitem()
            sort_data(v,k)
            j+=1
        else:
            time.sleep(1)
    q.join()
    while len(data) > 0:
        print("[%d/%d]" %(j,n))
        k,v = data.popitem()
        sort_data(v,k)
        j+=1
        

except KeyboardInterrupt:
    sys.exit(1)

code for recognition training:

from glob import glob
import cv2
import numpy as np
import pickle
from sklearn.preprocessing import LabelBinarizer
from sklearn.model_selection import train_test_split
from keras.models import Sequential
from keras.layers.convolutional import Conv2D, MaxPooling2D
from keras.layers.core import Flatten, Dense, Dropout
from keras.callbacks import EarlyStopping
import os
data = []
labels = []

    
os.makedirs("/app/model",exist_ok=True)

dirs = glob("/app/train/*/")
dirs = list(filter(lambda x: 'temp' != x.split('/')[-2],dirs))

for j,d in enumerate(dirs):
    l = d.split("/")[-2]
    files = glob(d+"/*")
    for k,f in enumerate(files):
        print('\r[dir: %d/%d file:%d/%d]' %(j,len(dirs),k,len(files)),end='')
        img = cv2.imread(f)
        img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
        img = cv2.resize(img, (20, 20))
        img = np.expand_dims(img, axis=2)
        data.append(img)
        labels.append(l)
print('[*] done')

data = np.array(data, dtype="float") / 255.0

print("[*] Training for %d chars" % (len(dirs)))

(X_train, X_test, Y_train, Y_test) = train_test_split(data, np.array(labels), test_size=0.40, random_state=0)

lb = LabelBinarizer().fit(Y_train)
Y_train = lb.transform(Y_train)
Y_test = lb.transform(Y_test)

print("[*] writing labels")
with open("/app/model/labels.dat", "wb") as f:
    pickle.dump(lb,f)
print("[*] done!")

print("[*] building model")

early_stop = EarlyStopping(monitor='val_loss', min_delta=0, patience=3, verbose=1, mode='auto')

model = Sequential()

model.add(Conv2D(20, (5, 5), padding="same", input_shape=(20, 20, 1), activation="relu"))
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))

model.add(Conv2D(50, (5, 5), padding="same", activation="relu"))
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))

model.add(Flatten())
model.add(Dense(500, activation="relu"))
model.add(Dropout(0.5, name='dropout_1'))

model.add(Dense(len(dirs), activation="softmax"))

model.compile(loss="categorical_crossentropy", optimizer="adam", metrics=["accuracy"])
print("[*] done!")

print("[*] start training")
model.fit(X_train, Y_train, validation_data=(X_test, Y_test), batch_size=len(dirs), epochs=10, verbose=1,callbacks=[early_stop  ])

print("[*] done!")

print("[*] saving model")
model.save("/app/model/model.hdf5")
print("[*] done!")

code for segmentation data collection:

import requests
import os
import time
import cv2
import base64
from PIL import Image
import string
from threading import Thread, Lock
import sys
from queue import Queue
import time
import random

def collect():
    s = requests.Session()
    url = "http://hax1.allesctf.net:9200"
    base="/app/segmentation/train/temp"
    r = s.get(url+"/captcha/0")
    imb64 = r.text.split('<form action="" method="post">')[1].split('<img src="')[1].split('">')[0].split("base64,")[1]
    im = base64.decodebytes(imb64.encode())
    try:
        os.makedirs(base+"/failed")
    except:
        pass
    base_name = ''.join(random.choices(string.ascii_lowercase,k=5))
    name = base+"/%s.bmp" %(base_name)
    with open(name, "wb") as f:
        f.write(im)

    r = s.post(url+"/captcha/0",data={'0':''})
    solution =r.text.split('The solution would have been <b>')[1].split('</b>')[0]

    image = cv2.imread(name)
    gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
    gray = cv2.copyMakeBorder(gray, 8, 8, 8, 8, cv2.BORDER_CONSTANT,value=(255,255,255))
    gray[gray > 40] = 255

    #image = cv2.copyMakeBorder(image, 8, 8, 8, 8, cv2.BORDER_CONSTANT,value=(255,255,255))
    thresh = cv2.threshold(gray, 0, 255, cv2.THRESH_BINARY_INV | cv2.THRESH_OTSU)[1]

    cnts = cv2.findContours(thresh.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)[0]
    
    regions = {}

    tolerance = 5
    last = -1

    for cnt in cnts:
        (x, y, w, h) = cv2.boundingRect(cnt)
        if w*h < 4:
            continue
        if x in regions:
            (x1, y1, w1, h1) = regions[x]
            x_ = min(x,x1)
            y_ = min(y,y1)
            ww_ = max(x+w,x1+w1)
            hh_ = max(y+h,y1+h1)
            regions[x]=(x_,y_,ww_-x_,hh_-y_)
            continue
        regions[x]=(x,y,w,h)
    for k in sorted(regions):
        (x, y, w, h) = regions[k]
        #cv2.rectangle(image, (x, y), (x+w, y+h), (255, 0, 0), 1)
        if last >= 0:
            (x1, y1, w1, h1) = regions[last]
            if ((abs(1-w/h) < 0.35 and w <8 and h <8) or (abs(1-w1/h1) < 0.35) and w1 <8 and h1 <8):
                if x < x1 + w1 +tolerance and  x + w +tolerance > x1:
                    x_ = min(x,x1)
                    y_ = min(y,y1)
                    ww_ = max(x+w,x1+w1)
                    hh_ = max(y+h,y1+h1)
                    regions[last]=(x_,y_,ww_-x_,hh_-y_)
                    del regions[k]
                    #cv2.rectangle(image, (x_,y_), (ww_,hh_), (255, 255, 0), 1)
                    continue
            elif x < x1 + w1 and x + w > x1:
                x_ = min(x,x1)
                y_ = min(y,y1)
                ww_ = max(x+w,x1+w1)
                hh_ = max(y+h,y1+h1)
                regions[last]=(x_,y_,ww_-x_,hh_-y_)
                del regions[k]
                #cv2.rectangle(image, (x_,y_), (ww_,hh_), (255, 255, 0), 1)
                continue


    #cv2.imwrite(name+".bmp", image)

    i = 0
    

    try:
        os.makedirs("/app/segmentation/train/1")
    except:
        pass
    try:
        os.makedirs("/app/segmentation/train/2")
    except:
        pass
    b=0
    bk =-1
    bn = 0
    for k in sorted(regions):
        

        x, y, w, h = regions[k]
        letter = gray[y-2:y + h+2, x-2 :x + w +2 ]
        name = "/app/segmentation/train/1/%c_%s.bmp" %(solution[i],''.join(random.choices(string.ascii_lowercase,k=4)))

        if w/h >b:
            b= w/h
            bk =k
            bn = name

        i+=1

        cv2.imwrite(name, letter)
    if len(solution) != len(regions.keys()):
        os.remove(bn)
        x, y, w, h = regions[bk]
        letter = gray[y-2:y + h+2, x-2 :x + w +2 ]
        name = "/app/segmentation/train/2/%c_%s.bmp" %(solution[i],''.join(random.choices(string.ascii_lowercase,k=4)))
        cv2.imwrite(name, letter)

        return
 
   
concurrent = 40
n = 10000
q = Queue(concurrent)
def worker():
    while True:    
        q.get()()
        q.task_done()
        time.sleep(0.05)

for i in range(concurrent):
    t = Thread(target=worker)
    t.daemon = True
    t.start()
try:
    for i in range(n):
        q.put(collect)
        print("[%d/%d]" %(i,n))
    while not q.empty():
        print("[%d/%d]" %(n-q.qsize(),n))
        time.sleep(0.1)
    q.join()
except KeyboardInterrupt:
    sys.exit(1)

code for segmentation data training:

from glob import glob
import cv2
import numpy as np
import pickle
from sklearn.preprocessing import LabelBinarizer
from sklearn.model_selection import train_test_split
from keras.models import Sequential
from keras.layers.convolutional import Conv2D, MaxPooling2D
from keras.layers.core import Flatten, Dense, Dropout
from keras.callbacks import EarlyStopping
import os
data = []
labels = []

    
os.makedirs("/app/segmentation/model",exist_ok=True)

dirs = glob("/app/segmentation/train/*/")
dirs = list(filter(lambda x: 'temp' != x.split('/')[-2],dirs))

for j,d in enumerate(dirs):
    l = d.split("/")[-2]
    files = glob(d+"/*")
    for k,f in enumerate(files):
        print('\r[dir: %d/%d file:%d/%d]' %(j,len(dirs),k,len(files)),end='')
        try:
            img = cv2.imread(f)
            
            height, width = img.shape[:2]
            img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
            w_ = round(20*(width/height))
            img = cv2.resize(img,(w_,20))
            img = cv2.copyMakeBorder(img, 0, 0, max((40-w_)//2,0), max((40-w_)//2,0), cv2.BORDER_CONSTANT,value=(255,255,255))
            img = cv2.resize(img, (40, 20))
            #cv2.imwrite(f+"_nn.bmp", img)
            img = np.expand_dims(img, axis=2)
            data.append(img)
            labels.append(l)
        except Exception as e:
            print(e)
            print(f)
            pass
print('[*] done')

data = np.array(data, dtype="float") / 255.0

print("[*] Training for %d chars" % (len(dirs)))

(X_train, X_test, Y_train, Y_test) = train_test_split(data, np.array(labels), test_size=0.40, random_state=0)

lb = LabelBinarizer().fit(Y_train)
Y_train = lb.transform(Y_train)
Y_test = lb.transform(Y_test)

print("[*] writing labels")
with open("/app/segmentation/model/labels.dat", "wb") as f:
    pickle.dump(lb,f)
print("[*] done!")

print("[*] building model")

early_stop = EarlyStopping(monitor='val_loss', min_delta=0, patience=3, verbose=1, mode='auto')

model = Sequential()

model.add(Conv2D(20, (5, 5), padding="same", input_shape=(20, 40, 1), activation="relu"))
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))

model.add(Conv2D(50, (5, 5), padding="same", activation="relu"))
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))

model.add(Flatten())
model.add(Dense(80, activation="relu"))
model.add(Dropout(0.25, name='dropout_1'))

model.add(Dense(len(dirs)-1, activation="sigmoid"))

model.compile(loss="binary_crossentropy", optimizer="adam", metrics=["accuracy"])
print("[*] done!")

print("[*] start training")
model.fit(X_train, Y_train, validation_data=(X_test, Y_test), batch_size=len(dirs), epochs=10, verbose=1,callbacks=[early_stop ])

print("[*] done!")

print("[*] saving model")
model.save("/app/segmentation/model/model.hdf5")
print("[*] done!")

code for segmentation prediction:

from keras.models import load_model
import numpy as np
import cv2
import pickle

with open("/app/segmentation/model/labels.dat", "rb") as f:
    lb = pickle.load(f)

model = load_model("/app/segmentation/model/model.hdf5")

def is_two(img):
    height, width = img.shape[:2]
    w_ = round(20*(width/height))
    img = cv2.resize(img,(w_,20))
    img = cv2.copyMakeBorder(img, 0, 0, max((40-w_)//2,0), max((40-w_)//2,0), cv2.BORDER_CONSTANT,value=(255,255,255))

    img = cv2.resize(img, (40, 20))
    img = np.expand_dims(img, axis=2)
    img = np.expand_dims(img, axis=0)

    prediction = model.predict(img)
    c = lb.inverse_transform(prediction)[0]
    return int(c)

Mitigation

• use reCAPTCHA or something similar

Flag

CSCG{Y0UR_B0T_S0LV3D_THE_CAPTCHA}