bitbake: toasterui: disable autocommit for build logging
This patch disables autocommit for inserting build data, effectively updating all build data in a single transaction. This is a purely performance improvement patch, as the transaction will always be commited. Similar manual transaction handling in the layer source update method. Added feedback messages during update method. [YOCTO #7140] (Bitbake rev: 3978c819e797f857235499a4b8ec238134f1c028) Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
parent
6de6739c4c
commit
cb23e3043b
|
@ -35,6 +35,7 @@ from bb.msg import BBLogFormatter as format
|
|||
from django.db import models
|
||||
import logging
|
||||
|
||||
from django.db import transaction
|
||||
|
||||
logger = logging.getLogger("BitBake")
|
||||
|
||||
|
@ -606,7 +607,9 @@ class BuildInfoHelper(object):
|
|||
self.internal_state = {}
|
||||
self.internal_state['taskdata'] = {}
|
||||
self.task_order = 0
|
||||
self.autocommit_step = 1
|
||||
self.server = server
|
||||
transaction.set_autocommit(False)
|
||||
self.orm_wrapper = ORMWrapper()
|
||||
self.has_build_history = has_build_history
|
||||
self.tmp_dir = self.server.runCommand(["getVariable", "TMPDIR"])[0]
|
||||
|
@ -926,6 +929,12 @@ class BuildInfoHelper(object):
|
|||
task_information['outcome'] = Task.OUTCOME_FAILED
|
||||
del self.internal_state['taskdata'][identifier]
|
||||
|
||||
# we force a sync point here, to get the progress bar to show
|
||||
if self.autocommit_step % 3 == 0:
|
||||
transaction.set_autocommit(True)
|
||||
transaction.set_autocommit(False)
|
||||
self.autocommit_step += 1
|
||||
|
||||
self.orm_wrapper.get_update_task_object(task_information, True) # must exist
|
||||
|
||||
|
||||
|
@ -1197,3 +1206,5 @@ class BuildInfoHelper(object):
|
|||
# we have no build, and we still have events; something amazingly wrong happend
|
||||
for event in self.internal_state['backlog']:
|
||||
logger.error("UNSAVED log: %s", event.msg)
|
||||
|
||||
transaction.set_autocommit(True)
|
||||
|
|
|
@ -664,6 +664,7 @@ class LayerIndexLayerSource(LayerSource):
|
|||
"""
|
||||
assert self.apiurl is not None
|
||||
from django.db import IntegrityError
|
||||
from django.db import transaction
|
||||
|
||||
import httplib, urlparse, json
|
||||
import os
|
||||
|
@ -715,6 +716,7 @@ class LayerIndexLayerSource(LayerSource):
|
|||
# update branches; only those that we already have names listed in the Releases table
|
||||
whitelist_branch_names = map(lambda x: x.branch_name, Release.objects.all())
|
||||
|
||||
print "Fetching branches"
|
||||
branches_info = _get_json_response(apilinks['branches']
|
||||
+ "?filter=name:%s" % "OR".join(whitelist_branch_names))
|
||||
for bi in branches_info:
|
||||
|
@ -727,6 +729,7 @@ class LayerIndexLayerSource(LayerSource):
|
|||
|
||||
# update layers
|
||||
layers_info = _get_json_response(apilinks['layerItems'])
|
||||
transaction.set_autocommit(False)
|
||||
for li in layers_info:
|
||||
l, created = Layer.objects.get_or_create(layer_source = self, name = li['name'])
|
||||
l.up_id = li['id']
|
||||
|
@ -738,11 +741,15 @@ class LayerIndexLayerSource(LayerSource):
|
|||
l.summary = li['summary']
|
||||
l.description = li['description']
|
||||
l.save()
|
||||
transaction.set_autocommit(True)
|
||||
|
||||
# update layerbranches/layer_versions
|
||||
print "Fetching layer information"
|
||||
layerbranches_info = _get_json_response(apilinks['layerBranches']
|
||||
+ "?filter=branch:%s" % "OR".join(map(lambda x: str(x.up_id), [i for i in Branch.objects.filter(layer_source = self) if i.up_id is not None] ))
|
||||
)
|
||||
|
||||
transaction.set_autocommit(False)
|
||||
for lbi in layerbranches_info:
|
||||
lv, created = Layer_Version.objects.get_or_create(layer_source = self,
|
||||
up_id = lbi['id'],
|
||||
|
@ -755,10 +762,12 @@ class LayerIndexLayerSource(LayerSource):
|
|||
lv.commit = lbi['actual_branch']
|
||||
lv.dirpath = lbi['vcs_subdir']
|
||||
lv.save()
|
||||
transaction.set_autocommit(True)
|
||||
|
||||
# update layer dependencies
|
||||
layerdependencies_info = _get_json_response(apilinks['layerDependencies'])
|
||||
dependlist = {}
|
||||
transaction.set_autocommit(False)
|
||||
for ldi in layerdependencies_info:
|
||||
try:
|
||||
lv = Layer_Version.objects.get(layer_source = self, up_id = ldi['layerbranch'])
|
||||
|
@ -777,23 +786,29 @@ class LayerIndexLayerSource(LayerSource):
|
|||
LayerVersionDependency.objects.filter(layer_version = lv).delete()
|
||||
for lvd in dependlist[lv]:
|
||||
LayerVersionDependency.objects.get_or_create(layer_version = lv, depends_on = lvd)
|
||||
transaction.set_autocommit(True)
|
||||
|
||||
|
||||
# update machines
|
||||
print "Fetching machine information"
|
||||
machines_info = _get_json_response(apilinks['machines']
|
||||
+ "?filter=layerbranch:%s" % "OR".join(map(lambda x: str(x.up_id), Layer_Version.objects.filter(layer_source = self)))
|
||||
)
|
||||
transaction.set_autocommit(False)
|
||||
for mi in machines_info:
|
||||
mo, created = Machine.objects.get_or_create(layer_source = self, up_id = mi['id'], layer_version = Layer_Version.objects.get(layer_source = self, up_id = mi['layerbranch']))
|
||||
mo.up_date = mi['updated']
|
||||
mo.name = mi['name']
|
||||
mo.description = mi['description']
|
||||
mo.save()
|
||||
transaction.set_autocommit(True)
|
||||
|
||||
# update recipes; paginate by layer version / layer branch
|
||||
print "Fetching target information"
|
||||
recipes_info = _get_json_response(apilinks['recipes']
|
||||
+ "?filter=layerbranch:%s" % "OR".join(map(lambda x: str(x.up_id), Layer_Version.objects.filter(layer_source = self)))
|
||||
)
|
||||
transaction.set_autocommit(False)
|
||||
for ri in recipes_info:
|
||||
try:
|
||||
ro, created = Recipe.objects.get_or_create(layer_source = self, up_id = ri['id'], layer_version = Layer_Version.objects.get(layer_source = self, up_id = ri['layerbranch']))
|
||||
|
@ -809,8 +824,9 @@ class LayerIndexLayerSource(LayerSource):
|
|||
ro.file_path = ri['filepath'] + "/" + ri['filename']
|
||||
ro.save()
|
||||
except:
|
||||
print "Duplicate Recipe, ignoring: ", vars(ro)
|
||||
#print "Duplicate Recipe, ignoring: ", vars(ro)
|
||||
pass
|
||||
transaction.set_autocommit(True)
|
||||
pass
|
||||
|
||||
class BitbakeVersion(models.Model):
|
||||
|
|
Loading…
Reference in New Issue