Skip to content

Commit

Permalink
added OCI format to scan_docker_image_lite
Browse files Browse the repository at this point in the history
  • Loading branch information
@mkumykov
mkumykov committed Jul 3, 2024
1 parent 4f4428a commit bd36209
Showing 1 changed file with 99 additions and 20 deletions.
119 changes: 99 additions & 20 deletions examples/scan_docker_image_lite.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
Alternative version if Docker image layer by layer scan.
This program will download docker image and scan it into Blackduck server layer by layer
This program will download docker image and scan it into Black Duck server layer by layer
Each layer will be scanned as a separate scan with a signature scan.
Layers in the container images could be grouped into groups of contiguous layers.
Expand Down Expand Up @@ -50,16 +50,16 @@
optional arguments:
-h, --help show this help message and exit
--grouping GROUPING Group layers into user defined provect versions (can't be used with --base-image)
--grouping GROUPING Group layers into user defined project versions (can't be used with --base-image)
--base-image BASE_IMAGE
Use base image spec to determine base image/layers (can't be used with --grouping or
--dockerfile)
--dockerfile DOCKERFILE
Use Dockerfile to determine base image/layers (can't be used with --grouping or ---base-image)
--project-name Specify project name (default is container image spec)
--project-verson Specify project version (default is container image tag/version)
--project-version Specify project version (default is container image tag/version)
--detect-options DETECT_OPTIONS
Extra detect options to be passed directlyto the detect
Extra detect options to be passed directly to the detect
Using --detect-options
Expand Down Expand Up @@ -188,12 +188,24 @@ def read_config(self):
with open(configFile) as fp:
data = json.load(fp)
return data

def read_oci_layout(self):
oci_layout_file = self.imagedir + "/" + 'oci-layout'
if os.path.exists(oci_layout_file) and os.path.isfile(oci_layout_file):
with open(oci_layout_file) as fp:
data = json.load(fp)
return data
else:
return None


class Detector():
def __init__(self, hub):
# self.detecturl = 'https://blackducksoftware.github.io/hub-detect/hub-detect.sh'
# self.detecturl = 'https://detect.synopsys.com/detect.sh'
self.detecturl = 'https://detect.synopsys.com/detect7.sh'
# self.detecturl = 'https://detect.synopsys.com/detect7.sh'
# self.detecturl = 'https://detect.synopsys.com/detect8.sh'
self.detecturl = 'https://detect.synopsys.com/detect9.sh'
self.baseurl = hub.config['baseurl']
self.filename = '/tmp/hub-detect.sh'
self.token=hub.config['api_token']
Expand Down Expand Up @@ -240,6 +252,7 @@ def __init__(
if detect_options:
self.extra_options = detect_options.split(" ")
print ("<--{}-->".format(self.grouping))
self.binary = False

def prepare_container_image(self):
self.docker.initdir()
Expand All @@ -262,6 +275,7 @@ def prepare_container_image(self):
self.grouping = history_grouping
self.docker.save_container_image(self.container_image_name)
self.docker.unravel_container()
self.oci_layout = self.docker.read_oci_layout()

def process_container_image_by_user_defined_groups(self):
self.manifest = self.docker.read_manifest()
Expand Down Expand Up @@ -333,25 +347,86 @@ def process_container_image_by_base_image_info(self):
num = num + 1
print (json.dumps(self.layers, indent=4))

def process_oci_container_image_by_user_defined_groups(self):
self.manifest = self.docker.read_manifest()
self.config = self.docker.read_config()

self.layers = self.config['history']
tagged_layers = [x for x in self.layers if '_group_end' in x.get('created_by')]
groups = {re.search('echo (.+?)_group_end', str(x['created_by'])).group(1): self.layers.index(x) for x in tagged_layers}
print (groups)
layer_paths = self.manifest[0]['Layers'].copy()
empty_layers = [x for x in self.layers if x.get('empty_layer', False)]
print(f"Total layers: {len(self.layers)} total paths: {len(layer_paths)} empty layers: {len(empty_layers)}")

assert len(self.layers) == len(layer_paths) + len(empty_layers), "Something is wrong with this image, Layer math does not add up."

for layer in self.layers:
layer['index'] = self.layers.index(layer)
if self.grouping:
layer['group_name'] = self.get_group_name(groups, layer['index'])
layer['project_version'] = "{}_{}".format(self.project_version,layer['group_name'])
layer['name'] = "{}_{}_{}_layer_{}".format(self.project_name,self.project_version,layer['group_name'],str(layer['index']))
else:
layer['project_version'] = self.project_version
layer['name'] = self.project_name + "_" + self.project_version + "_layer_" + str(layer['index'])
layer['project_name'] = self.project_name
if not layer.get('empty_layer', False):
layer['path'] = layer_paths.pop(0)
print (json.dumps(self.layers, indent=4))

def get_group_name(self, groups, index):
group_name = 'undefined'
for group, value in groups.items():
if index <= value:
group_name = group
break
return group_name

def process_oci_container_image_by_base_image_info(self):
print ("Processing by BAse Image not supported for OCI images")
sys.exit(1)
pass

def process_container_image(self):
if self.oci_layout:
self.process_oci_container_image()
else:
self.process_docker_container_image()

def process_docker_container_image(self):
if self.grouping:
self.process_container_image_by_user_defined_groups()
else:
self.process_container_image_by_base_image_info()

def process_oci_container_image(self):
if self.grouping:
self.process_oci_container_image_by_user_defined_groups()
else:
self.process_oci_container_image_by_base_image_info()

def submit_layer_scans(self):
for layer in self.layers:
options = []
options.append('--detect.project.name={}'.format(layer['project_name']))
options.append('--detect.project.version.name="{}"'.format(layer['project_version']))
# options.append('--detect.blackduck.signature.scanner.disabled=false')
options.append('--detect.code.location.name={}_{}_code_{}'.format(layer['name'],self.image_version,layer['path']))
options.append('--detect.source.path={}/{}'.format(self.docker.imagedir, layer['path'].split('/')[0]))
if self.base_image or self.grouping or self.dockerfile:
options.extend(self.adorn_extra_options(layer))
else:
options.extend(self.extra_options)
self.hub_detect.detect_run(options)
if not layer.get('empty_layer', False):
options = []
options.append('--detect.project.name={}'.format(layer['project_name']))
options.append('--detect.project.version.name="{}"'.format(layer['project_version']))
options.append('--detect.code.location.name={}_{}_code_{}'.format(layer['name'],self.image_version,layer['path']))
if self.binary:
options.append('--detect.tools=BINARY_SCAN')
options.append('--detect.binary.scan.file.path={}/{}'.format(self.docker.imagedir, layer['path']))
else:
options.append('--detect.tools=SIGNATURE_SCAN')
if self.oci_layout:
options.append('--detect.source.path={}/{}'.format(self.docker.imagedir, layer['path']))
else:
options.append('--detect.source.path={}/{}'.format(self.docker.imagedir, layer['path'].split('/')[0]))
if self.base_image or self.grouping or self.dockerfile:
options.extend(self.adorn_extra_options(layer))
else:
options.extend(self.extra_options)
self.hub_detect.detect_run(options)

def adorn_extra_options(self, layer):
result = list()
Expand Down Expand Up @@ -401,7 +476,7 @@ def get_base_layers(self):

def scan_container_image(
imagespec, grouping=None, base_image=None, dockerfile=None,
project_name=None, project_version=None, detect_options=None):
project_name=None, project_version=None, detect_options=None, binary=False):

hub = HubInstance()
scanner = ContainerImageScanner(
Expand All @@ -416,6 +491,8 @@ def scan_container_image(
scanner.grouping = '1024:everything'
else:
scanner.base_layers = scanner.get_base_layers()
if binary:
scanner.binary = True
scanner.prepare_container_image()
scanner.process_container_image()
scanner.submit_layer_scans()
Expand All @@ -429,12 +506,13 @@ def main(argv=None):

parser = ArgumentParser()
parser.add_argument('imagespec', help="Container image tag, e.g. repository/imagename:version")
parser.add_argument('--grouping',default=None, type=str, help="Group layers into user defined provect versions (can't be used with --base-image)")
parser.add_argument('--grouping',default=None, type=str, help="Group layers into user defined project versions (can't be used with --base-image)")
parser.add_argument('--base-image',default=None, type=str, help="Use base image spec to determine base image/layers (can't be used with --grouping or --dockerfile)")
parser.add_argument('--dockerfile',default=None, type=str, help="Use Dockerfile to determine base image/layers (can't be used with --grouping or ---base-image)")
parser.add_argument('--project-name',default=None, type=str, help="Specify project name (default is container image spec)")
parser.add_argument('--project-version',default=None, type=str, help="Specify project version (default is container image tag/version)")
parser.add_argument('--detect-options',default=None, type=str, help="Extra detect options to be passed directlyto the detect")
parser.add_argument('--detect-options',default=None, type=str, help="Extra detect options to be passed directly to the detect")
parser.add_argument('--binary', action='store_true', help="Use Binary Scan instead of signature scan")

args = parser.parse_args()

Expand All @@ -459,7 +537,8 @@ def main(argv=None):
args.dockerfile,
args.project_name,
args.project_version,
args.detect_options)
args.detect_options,
args.binary)


if __name__ == "__main__":
Expand Down

0 comments on commit bd36209

Please sign in to comment.