WholeCellKB.public.helpers

1 ''' 2 Whole-cell helper functions 3 4 Author: Jonathan Karr, jkarr@stanford.edu 5 Affiliation: Covert Lab, Department of Bioengineering, Stanford University 6 Last updated: 2012-07-17 7 ''' 8 9 from __future__ import unicode_literals 10 from BeautifulSoup import BeautifulStoneSoup 11 from copy import deepcopy 12 from dateutil.tz import tzlocal 13 from django.contrib.auth.models import User 14 from django.core import serializers 15 from django.core.exceptions import ValidationError, ObjectDoesNotExist 16 from django.core.urlresolvers import reverse 17 from django.db import IntegrityError 18 from django.db.models import get_app, get_models, get_model 19 from django.db.models.fields import AutoField, BigIntegerField, IntegerField, PositiveIntegerField, PositiveSmallIntegerField, SmallIntegerField, BooleanField, NullBooleanField, DecimalField, FloatField, CharField, CommaSeparatedIntegerField, EmailField, FilePathField, GenericIPAddressField, IPAddressField, SlugField, URLField, TextField, DateField, DateTimeField, TimeField, NOT_PROVIDED 20 from django.db.models.fields.related import OneToOneField, RelatedObject, ManyToManyField, ForeignKey 21 from django.db.models.query import EmptyQuerySet 22 from django.http import Http404, HttpResponse, HttpResponseBadRequest 23 from django.shortcuts import render_to_response, get_object_or_404 24 from django.template import Context, RequestContext, loader 25 from django.template.loader import get_template 26 from django.utils import simplejson 27 from django.utils.html import strip_tags 28 from django.utils.text import capfirst 29 from odict import odict 30 from openpyxl import Workbook, load_workbook 31 from openpyxl.cell import Cell, get_column_letter 32 from openpyxl.shared.date_time import SharedDate, CALENDAR_WINDOWS_1900 33 from openpyxl.style import NumberFormat, Border, Color, HashableObject, Protection, Alignment 34 from openpyxl.writer.styles import StyleWriter 35 from public.templatetags.templatetags import ceil 36 from public.models import Entry, Species, SpeciesComponent, Reference, Chromosome, format_list_html, format_evidence, Evidence, EvidencedEntryData 37 from public.models import EntryData, EntryBooleanData, EntryCharData, EntryFloatData, EntryPositiveFloatData, EntryTextData 38 from pygments.filter import Filter 39 from pygments.formatters import HtmlFormatter 40 from pygments.style import Style 41 from pygments.token import Token 42 from StringIO import StringIO 43 from subprocess import Popen, PIPE 44 from xml.dom.minidom import Document 45 import datetime 46 import htmlentitydefs 47 import inspect 48 import math 49 import os 50 import pygments 51 import re 52 import settings 53 import sys 54 import tempfile 55 import time 56 import xhtml2pdf.pisa as pisa 57

58 -class ELEMENT_MWS:

59 H = 1.0079 60 He = 4.0026 61 Li = 6.941 62 Be = 9.0122 63 B = 10.811 64 C = 12.0107 65 N = 14.0067 66 O = 15.9994 67 F = 18.9984 68 Ne = 20.1797 69 Na = 22.9897 70 Mg = 24.305 71 Al = 26.9815 72 Si = 28.0855 73 P = 30.9738 74 S = 32.065 75 Cl = 35.453 76 Ar = 39.948 77 K = 39.0983 78 Ca = 40.078 79 Sc = 44.9559 80 Ti = 47.867 81 V = 50.9415 82 Cr = 51.9961 83 Mn = 54.938 84 Fe = 55.845 85 Co = 58.9332 86 Ni = 58.6934 87 Cu = 63.546 88 Zn = 65.39 89 Ga = 69.723 90 Ge = 72.64 91 As = 74.9216 92 Se = 78.96 93 Br = 79.904 94 Kr = 83.8 95 Rb = 85.4678 96 Sr = 87.62 97 Y = 88.9059 98 Zr = 91.224 99 Nb = 92.9064 100 Mo = 95.94 101 Tc = 98 102 Ru = 101.07 103 Rh = 102.9055 104 Pd = 106.42 105 Ag = 107.8682 106 Cd = 112.411 107 In = 114.818 108 Sn = 118.71 109 Sb = 121.76 110 Te = 127.6 111 I = 126.9045 112 Xe = 131.293 113 Cs = 132.9055 114 Ba = 137.327 115 La = 138.9055 116 Ce = 140.116 117 Pr = 140.9077 118 Nd = 144.24 119 Pm = 145 120 Sm = 150.36 121 Eu = 151.964 122 Gd = 157.25 123 Tb = 158.9253 124 Dy = 162.5 125 Ho = 164.9303 126 Er = 167.259 127 Tm = 168.9342 128 Yb = 173.04 129 Lu = 174.967 130 Hf = 178.49 131 Ta = 180.9479 132 W = 183.84 133 Re = 186.207 134 Os = 190.23 135 Ir = 192.217 136 Pt = 195.078 137 Au = 196.9665 138 Hg = 200.59 139 Tl = 204.3833 140 Pb = 207.2 141 Bi = 208.9804 142 Po = 209 143 At = 210 144 Rn = 222 145 Fr = 223 146 Ra = 226 147 Ac = 227 148 Th = 232.0381 149 Pa = 231.0359 150 U = 238.0289 151 Np = 237 152 Pu = 244 153 Am = 243 154 Cm = 247 155 Bk = 247 156 Cf = 251 157 Es = 252 158 Fm = 257 159 Md = 258 160 No = 259 161 Lr = 262 162 Rf = 261 163 Db = 262 164 Sg = 266 165 Bh = 264 166 Hs = 277 167 Mt = 268

168 169 #Source: http://www.owczarzy.net/extinct.htm

170 -class ExtinctionCoefficient:

171 single_dna = { 172 'A': 15400, 173 'C': 7200, 174 'G': 11500, 175 'T': 8700, 176 } 177 single_rna = { 178 'A': 15400, 179 'C': 7200, 180 'G': 11500, 181 'U': 9900, 182 } 183 pairwise_dna = { 184 'A': {'A': 27400, 'C': 21200, 'G': 25000, 'T': 28000}, 185 'C': {'A': 21200, 'C': 14600, 'G': 18000, 'T': 15200}, 186 'G': {'A': 25200, 'C': 17600, 'G': 21600, 'T': 20000}, 187 'T': {'A': 23400, 'C': 16200, 'G': 19000, 'T': 16800}, 188 } 189 pairwise_rna = { 190 'A': {'A': 27400, 'C': 21000, 'G': 25000, 'U': 24000}, 191 'C': {'A': 21000, 'C': 14200, 'G': 17800, 'U': 16200}, 192 'G': {'A': 25200, 'C': 17400, 'G': 21600, 'U': 21200}, 193 'U': {'A': 24600, 'C': 17200, 'G': 20000, 'U': 19600}, 194 }

195 196 ''' 197 dipeptide instability weight value (DIWV) 198 1. http://ca.expasy.org/tools/protparam-doc.html 199 2. Kunchur Guruprasad, B.V.Bhasker Reddy and Madhusudan W.Pandit 200 (1990). Correlation between stability of a protein and its 201 dipeptide composition: a novel approach for predicting in vivo 202 stability of a protein from its primary sequence. Protein 203 Engineering 4(2):155-161. 204 '''

205 -class DipeptideInstabilityWeight:

206 value = { 207 "A": {"A": 1.00, "R": 1.00, "N": 1.00, "D": -7.49, "C": 44.94, "Q": 1.00, "E": 1.00, "G": 1.00, "H": -7.49, "I": 1.00, "L": 1.00, "K": 1.00, "M": 1.00, "F": 1.00, "P": 20.26, "S": 1.00, "T": 1.00, "W": 1.00, "Y": 1.00, "V": 1.00, }, 208 "R": {"A": 1.00, "R": 58.28, "N": 13.34, "D": 1.00, "C": 1.00, "Q": 20.26, "E": 1.00, "G": -7.49, "H": 20.26, "I": 1.00, "L": 1.00, "K": 1.00, "M": 1.00, "F": 1.00, "P": 20.26, "S": 44.94, "T": 1.00, "W": 58.28, "Y": -6.54, "V": 1.00, }, 209 "N": {"A": 1.00, "R": 1.00, "N": 1.00, "D": 1.00, "C": -1.88, "Q": -6.54, "E": 1.00, "G": -14.03, "H": 1.00, "I": 44.94, "L": 1.00, "K": 24.68, "M": 1.00, "F": -14.03, "P": -1.88, "S": 1.00, "T": -7.49, "W": -9.37, "Y": 1.00, "V": 1.00, }, 210 "D": {"A": 1.00, "R": -6.54, "N": 1.00, "D": 1.00, "C": 1.00, "Q": 1.00, "E": 1.00, "G": 1.00, "H": 1.00, "I": 1.00, "L": 1.00, "K": -7.49, "M": 1.00, "F": -6.54, "P": 1.00, "S": 20.26, "T": -14.03, "W": 1.00, "Y": 1.00, "V": 1.00, }, 211 "C": {"A": 1.00, "R": 1.00, "N": 1.00, "D": 20.26, "C": 1.00, "Q": -6.54, "E": 1.00, "G": 1.00, "H": 33.60, "I": 1.00, "L": 20.26, "K": 1.00, "M": 33.60, "F": 1.00, "P": 20.26, "S": 1.00, "T": 33.60, "W": 24.68, "Y": 1.00, "V": -6.54, }, 212 "Q": {"A": 1.00, "R": 1.00, "N": 1.00, "D": 20.26, "C": -6.54, "Q": 20.26, "E": 20.26, "G": 1.00, "H": 1.00, "I": 1.00, "L": 1.00, "K": 1.00, "M": 1.00, "F": -6.54, "P": 20.26, "S": 44.94, "T": 1.00, "W": 1.00, "Y": -6.54, "V": -6.54, }, 213 "E": {"A": 1.00, "R": 1.00, "N": 1.00, "D": 20.26, "C": 44.94, "Q": 20.26, "E": 33.60, "G": 1.00, "H": -6.54, "I": 20.26, "L": 1.00, "K": 1.00, "M": 1.00, "F": 1.00, "P": 20.26, "S": 20.26, "T": 1.00, "W": -14.03, "Y": 1.00, "V": 1.00, }, 214 "G": {"A": -7.49, "R": 1.00, "N": -7.49, "D": 1.00, "C": 1.00, "Q": 1.00, "E": -6.54, "G": 13.34, "H": 1.00, "I": -7.49, "L": 1.00, "K": -7.49, "M": 1.00, "F": 1.00, "P": 1.00, "S": 1.00, "T": -7.49, "W": 13.34, "Y": -7.49, "V": 1.00, }, 215 "H": {"A": 1.00, "R": 1.00, "N": 24.68, "D": 1.00, "C": 1.00, "Q": 1.00, "E": 1.00, "G": -9.37, "H": 1.00, "I": 44.94, "L": 1.00, "K": 24.68, "M": 1.00, "F": -9.37, "P": -1.88, "S": 1.00, "T": -6.54, "W": -1.88, "Y": 44.94, "V": 1.00, }, 216 "I": {"A": 1.00, "R": 1.00, "N": 1.00, "D": 1.00, "C": 1.00, "Q": 1.00, "E": 44.94, "G": 1.00, "H": 13.34, "I": 1.00, "L": 20.26, "K": -7.49, "M": 1.00, "F": 1.00, "P": -1.88, "S": 1.00, "T": 1.00, "W": 1.00, "Y": 1.00, "V": -7.49, }, 217 "L": {"A": 1.00, "R": 20.26, "N": 1.00, "D": 1.00, "C": 1.00, "Q": 33.60, "E": 1.00, "G": 1.00, "H": 1.00, "I": 1.00, "L": 1.00, "K": -7.49, "M": 1.00, "F": 1.00, "P": 20.26, "S": 1.00, "T": 1.00, "W": 24.68, "Y": 1.00, "V": 1.00, }, 218 "K": {"A": 1.00, "R": 33.60, "N": 1.00, "D": 1.00, "C": 1.00, "Q": 24.68, "E": 1.00, "G": -7.49, "H": 1.00, "I": -7.49, "L": -7.49, "K": 1.00, "M": 33.60, "F": 1.00, "P": -6.54, "S": 1.00, "T": 1.00, "W": 1.00, "Y": 1.00, "V": -7.49, }, 219 "M": {"A": 13.34, "R": -6.54, "N": 1.00, "D": 1.00, "C": 1.00, "Q": -6.54, "E": 1.00, "G": 1.00, "H": 58.28, "I": 1.00, "L": 1.00, "K": 1.00, "M": -1.88, "F": 1.00, "P": 44.94, "S": 44.94, "T": -1.88, "W": 1.00, "Y": 24.68, "V": 1.00, }, 220 "F": {"A": 1.00, "R": 1.00, "N": 1.00, "D": 13.34, "C": 1.00, "Q": 1.00, "E": 1.00, "G": 1.00, "H": 1.00, "I": 1.00, "L": 1.00, "K": -14.03, "M": 1.00, "F": 1.00, "P": 20.26, "S": 1.00, "T": 1.00, "W": 1.00, "Y": 33.60, "V": 1.00, }, 221 "P": {"A": 20.26, "R": -6.54, "N": 1.00, "D": -6.54, "C": -6.54, "Q": 20.26, "E": 18.38, "G": 1.00, "H": 1.00, "I": 1.00, "L": 1.00, "K": 1.00, "M": -6.54, "F": 20.26, "P": 20.26, "S": 20.26, "T": 1.00, "W": -1.88, "Y": 1.00, "V": 20.26, }, 222 "S": {"A": 1.00, "R": 20.26, "N": 1.00, "D": 1.00, "C": 33.60, "Q": 20.26, "E": 20.26, "G": 1.00, "H": 1.00, "I": 1.00, "L": 1.00, "K": 1.00, "M": 1.00, "F": 1.00, "P": 44.94, "S": 20.26, "T": 1.00, "W": 1.00, "Y": 1.00, "V": 1.00, }, 223 "T": {"A": 1.00, "R": 1.00, "N": -14.03, "D": 1.00, "C": 1.00, "Q": -6.54, "E": 20.26, "G": -7.49, "H": 1.00, "I": 1.00, "L": 1.00, "K": 1.00, "M": 1.00, "F": 13.34, "P": 1.00, "S": 1.00, "T": 1.00, "W": -14.03, "Y": 1.00, "V": 1.00, }, 224 "W": {"A": -14.03, "R": 1.00, "N": 13.34, "D": 1.00, "C": 1.00, "Q": 1.00, "E": 1.00, "G": -9.37, "H": 24.68, "I": 1.00, "L": 13.34, "K": 1.00, "M": 24.68, "F": 1.00, "P": 1.00, "S": 1.00, "T": -14.03, "W": 1.00, "Y": 1.00, "V": -7.49, }, 225 "Y": {"A": 24.68, "R": -15.91, "N": 1.00, "D": 24.68, "C": 1.00, "Q": 1.00, "E": -6.54, "G": -7.49, "H": 13.34, "I": 1.00, "L": 1.00, "K": 1.00, "M": 44.94, "F": 1.00, "P": 13.34, "S": 1.00, "T": -7.49, "W": -9.37, "Y": 13.34, "V": 1.00, }, 226 "V": {"A": 1.00, "R": 1.00, "N": 1.00, "D": -14.03, "C": 1.00, "Q": 1.00, "E": 1.00, "G": -7.49, "H": 1.00, "I": 1.00, "L": 1.00, "K": -1.88, "M": 1.00, "F": 1.00, "P": 20.26, "S": 1.00, "T": -7.49, "W": 1.00, "Y": -6.54, "V": 1.00, }, 227 }

228

229 -class Colors(HashableObject):

230 BLACK = 'FF000000' 231 BLUE = 'FF306EFF' 232 GREY1 = 'FFCCCCCC' 233 GREY2 = 'FFDDDDDD' 234 GREY3 = 'FF999999' 235 GREY4 = 'FFAAAAAA' 236 WHITE = 'FFFFFFFF' 237 238 __fields__ = ('index',) 239 __slots__ = __fields__ 240 __leaf__ = True 241

242 - def __init__(self, index):

243 super(Color, self).__init__() 244 self.index = index

245

246 -def getModels(superclass=Entry, is_public=False):

247 tmp = get_models(get_app('public')) 248 tmp2 = {} 249 for model in tmp: 250 if issubclass(model, superclass) and model._meta.concrete_entry_model and (not is_public or not (hasattr(model._meta, 'hide_public') and model._meta.hide_public)): 251 tmp2[model.__name__] = model 252 return tmp2

253

254 -def get_models_in_saving_order(superclass=Entry, is_public=False):

255 all_models = get_models(get_app('public')) 256 entry_models = [] 257 for model in all_models: 258 if issubclass(model, superclass) and model._meta.concrete_entry_model and (not is_public or not (hasattr(model._meta, 'hide_public') and model._meta.hide_public)): 259 entry_models.append(model) 260 261 return sorted(entry_models, cmp=model_save_order_comparator)

262

263 -def model_save_order_comparator(x, y):

264 if is_model_referenced(x, y): 265 return 1 266 267 if is_model_referenced(y, x): 268 return -1 269 270 return 0

271

272 -def is_model_referenced(model, referenced_model, checked_models=[]):

273 for field in model._meta.fields + model._meta.many_to_many: 274 if isinstance(field, (ForeignKey, ManyToManyField)) and not (isinstance(field, OneToOneField) and field.rel.parent_link) and issubclass(field.rel.to, Entry): 275 if issubclass(referenced_model, field.rel.to): 276 return True 277 278 for field in model._meta.fields + model._meta.many_to_many: 279 if isinstance(field, (ForeignKey, ManyToManyField)) and not (isinstance(field, OneToOneField) and field.rel.parent_link) and issubclass(field.rel.to, Entry): 280 if (field.rel.to not in checked_models) and is_model_referenced(field.rel.to, referenced_model, checked_models + [model]): 281 return True 282 return False

283

284 -def getModelsMetadata(superclass=Entry, is_public=False):

285 tmp = get_models(get_app('public')) 286 tmp2 = {} 287 for model in tmp: 288 if issubclass(model, superclass) and model._meta.concrete_entry_model and (not is_public or not (hasattr(model._meta, 'hide_public') and model._meta.hide_public)): 289 tmp2[model.__name__] = model._meta 290 return tmp2

291

292 -def getObjectTypes(superclass=Entry, is_public=False):

293 tmp = get_models(get_app('public')) 294 tmp2 = [] 295 for model in tmp: 296 if issubclass(model, superclass) and model._meta.concrete_entry_model and (not is_public or not (hasattr(model._meta, 'hide_public') and model._meta.hide_public)): 297 tmp2.append(model.__name__) 298 tmp2.sort() 299 return tmp2

300

301 -def getModel(str):

302 tmp = get_models(get_app('public')) 303 tmp2 = [] 304 for model in tmp: 305 if issubclass(model, Entry) and model._meta.concrete_entry_model and model.__name__ == str: 306 return model 307 return

308

309 -def getModelByVerboseName(str):

310 tmp = get_models(get_app('public')) 311 tmp2 = [] 312 for model in tmp: 313 if issubclass(model, Entry) and model._meta.concrete_entry_model and model.verbose_name == str: 314 return model 315 return

316

317 -def getModelByVerboseNamePlural(str):

318 tmp = get_models(get_app('public')) 319 tmp2 = [] 320 for model in tmp: 321 if issubclass(model, Entry) and model._meta.concrete_entry_model and model._meta.verbose_name_plural == str: 322 return model 323 return

324

325 -def getModelAdmin(str):

326 app_ = __import__('public') 327 admins_ = getattr(app_, 'admin') 328 if hasattr(admins_, str + 'Admin'): 329 return getattr(admins_, str + 'Admin') 330 return None

331

332 -def getModelDataFields(model, auto_created=True, metadata=True):

333 unordered_fields = [] 334 for field in model._meta.fields + model._meta.many_to_many: 335 if not (field.name in ['species', 'model_type'] or (isinstance(field, OneToOneField) and field.rel.parent_link)) and (not field.auto_created or auto_created): 336 unordered_fields.append(field) 337 338 ordered_fields = [] 339 for field in model._meta.field_list: 340 if not metadata and field in ['id', 'created_user', 'last_updated_user']: 341 continue 342 343 field = model._meta.get_field_by_name(field)[0] 344 if not field.auto_created or auto_created: 345 ordered_fields.append(field) 346 347 if metadata and (len(unordered_fields) != len(ordered_fields) or set(unordered_fields) != set(ordered_fields)): 348 print str(set(unordered_fields).difference(ordered_fields)) 349 print str(set(ordered_fields).difference(unordered_fields)) 350 raise Http404 351 352 return ordered_fields

353

354 -def getModelDataFieldNames(model):

355 fields = [] 356 for field in model._meta.fields + model._meta.many_to_many: 357 if not (field.name == 'model_type' or (isinstance(field, OneToOneField) and field.rel.parent_link)): 358 fields.append(field.name) 359 return fields

360

361 -def is_entrydata_model(cls):

362 return inspect.isclass(cls) and issubclass(cls, EntryData)

363

364 -def getEntryDatas():

365 return inspect.getmembers(sys.modules['public.models'], is_entrydata_model)

366

367 -def getEntry(species_wid = None, wid = None):

368 try: 369 species_id = Species.objects.values('id').get(wid = species_wid)['id'] 370 except ObjectDoesNotExist: 371 return None 372 373 try: 374 if species_wid == wid: 375 return Species.objects.get(wid=wid) 376 else: 377 tmp = SpeciesComponent.objects.get(species__id = species_id, wid=wid) 378 return getModel(tmp.model_type).objects.select_related(depth=2).get(id=tmp.id) 379 except ObjectDoesNotExist: 380 return None

381

382 -def html_to_ascii(s):

383 return strip_tags(unicode(BeautifulStoneSoup(s, convertEntities=BeautifulStoneSoup.ALL_ENTITIES)).replace(" ", " "))

384 385 ''' 386 Element order preserved 387 From http://www.peterbe.com/plog/uniqifiers-benchmark 388 '''

389 -def uniqueUnsorted(seq):

390 seen = set() 391 return [x for x in seq if x not in seen and not seen.add(x)]

392 393 ''' 394 Element order not preserved 395 From http://www.peterbe.com/plog/uniqifiers-benchmark 396 '''

397 -def uniqueSorted(seq):

398 return {}.fromkeys(seq).keys()

399

400 -def render_queryset_to_response(request = [], queryset = EmptyQuerySet(), models = [], templateFile = '', data = {}, species_wid=None):

401 format = request.GET.get('format', 'html') 402 403 if species_wid is not None and species_wid != '': 404 species = Species.objects.get(wid=species_wid) 405 else: 406 species = Species.objects.all() 407 if len(species) > 0: 408 species = species[0] 409 else: 410 species = None 411 412 data['species'] = species 413 data['queryset'] = queryset 414 data['request'] = request 415 data['queryargs'] = {} 416 for key, val in request.GET.iterlists(): 417 data['queryargs'][key] = val 418 419 if format == 'html': 420 data['is_pdf'] = False 421 data['pdfstyles'] = '' 422 data['species_list'] = Species.objects.all() 423 data['modelmetadatas'] = getModelsMetadata(SpeciesComponent, request.user.is_anonymous()) 424 data['modelnames'] = getObjectTypes(SpeciesComponent, request.user.is_anonymous()) 425 data['last_updated_date'] = datetime.datetime.fromtimestamp(os.path.getmtime(settings.TEMPLATE_DIRS[0] + '/' + templateFile)) 426 427 return render_to_response(templateFile, data, context_instance = RequestContext(request)) 428 elif format == 'bib': 429 response = HttpResponse( 430 write_bibtex(species, queryset), 431 mimetype = "application/x-bibtex; charset=UTF-8", 432 content_type = "application/x-bibtex; charset=UTF-8") 433 response['Content-Disposition'] = "attachment; filename=data.bib" 434 elif format == 'json': 435 objects = [] 436 for obj in queryset: 437 objDict = convert_modelobject_to_stdobject(obj, request.user.is_anonymous()) 438 objDict['model'] = obj.__class__.__name__ 439 objects.append(objDict) 440 441 now = datetime.datetime.now(tzlocal()) 442 json = odict() 443 json['title'] = '%s WholeCellKB' % species.name 444 json['comments'] = 'Generated by %s on %s at %s' % ('WholeCellKB', now.isoformat(), settings.ROOT_URL + reverse('public.views.exportData', kwargs={'species_wid': species.wid})) 445 json['copyright'] = '%s %s' % (now.year, 'Covert Lab, Department of Bioengineering, Stanford University') 446 json['data'] = objects 447 response = HttpResponse( 448 simplejson.dumps(json, indent=2, ensure_ascii=False, encoding='utf-8'), 449 mimetype = "application/json; charset=UTF-8", 450 content_type = "application/json; charset=UTF-8") 451 response['Content-Disposition'] = "attachment; filename=data.json" 452 elif format == 'pdf': 453 data['is_pdf'] = True 454 data['pdfstyles'] = '' 455 data['species_list'] = Species.objects.all() 456 data['modelmetadatas'] = getModelsMetadata(SpeciesComponent, request.user.is_anonymous()) 457 data['modelnames'] = getObjectTypes(SpeciesComponent, request.user.is_anonymous()) 458 459 for fileName in ['styles', 'styles.print', 'styles.pdf']: 460 f = open(settings.STATICFILES_DIRS[0] + '/public/css/' + fileName + '.css', 'r') 461 data['pdfstyles'] += f.read() 462 f.close() 463 464 response = HttpResponse( 465 mimetype = 'application/pdf', 466 content_type = 'application/pdf') 467 response['Content-Disposition'] = "attachment; filename=data.pdf" 468 469 template = get_template(templateFile) 470 471 pdf = pisa.CreatePDF( 472 src = template.render(Context(data)), 473 dest = response) 474 475 if not pdf.err: 476 return response 477 return Http404 478 elif format == 'xlsx': 479 #write work book 480 wb = writeExcel(species, queryset, models, request.user.is_anonymous()) 481 482 #save to string 483 result = StringIO() 484 wb.save(filename = result) 485 486 #generate HttpResponse 487 response = HttpResponse( 488 result.getvalue(), 489 mimetype = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", 490 content_type = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet") 491 response['Content-Disposition'] = "attachment; filename=data.xlsx" 492 elif format == 'xml': 493 doc = Document() 494 495 now = datetime.datetime.now(tzlocal()) 496 comment = doc.createComment('\n%s WholeCellKB\nGenerated by %s on %s at %s\n%s %s %s\n' % ( 497 species.name, 498 'WholeCellKB', now.isoformat(), 499 settings.ROOT_URL + reverse('public.views.exportData', kwargs={'species_wid': species.wid}), 500 html_to_ascii('©'), now.year, 'Covert Lab, Department of Bioengineering, Stanford University', 501 )) 502 doc.appendChild(comment) 503 504 objects = doc.createElement('objects') 505 doc.appendChild(objects) 506 507 for obj in queryset: 508 objects.appendChild(convert_modelobject_to_xml(obj, doc, request.user.is_anonymous())) 509 510 response = HttpResponse( 511 doc.toprettyxml(indent=" "*2, encoding = 'utf-8'), 512 mimetype = "application/xml; charset=UTF-8", 513 content_type = "application/xml; charset=UTF-8") 514 response['Content-Disposition'] = "attachment; filename=data.xml" 515 else: 516 t = loader.get_template('public/error.html') 517 data['type'] = 500 518 data['message'] = '"%s" is not a supported export format.' % format 519 c = Context(data) 520 response = HttpResponseBadRequest( 521 t.render(c), 522 mimetype = 'text/html; charset=UTF-8', 523 content_type = 'text/html; charset=UTF-8') 524 525 return response 526

527 -def writeExcel(species, queryset, modelArr, is_user_anonymous):

528 #sort entry models by name 529 modelDict = {} 530 model_types = [] 531 objectDict = {} 532 for model in modelArr: 533 model_types.append(model.__name__) 534 modelDict[model.__name__] = model 535 objectDict[model.__name__] = [] 536 model_types.sort() 537 538 #organize entries by model 539 for obj in queryset: 540 objectDict[obj.__class__.__name__].append(obj) 541 542 #create workbook 543 wb = Workbook() 544 wb.remove_sheet(wb.get_active_sheet()) 545 546 #meta data 547 now = datetime.datetime.now(tzlocal()) 548 wb.properties.creator = 'Generated by WholeCellKB on %s at %s' % ( 549 now.isoformat(), 550 settings.ROOT_URL + reverse('public.views.exportData', kwargs={'species_wid': species.wid}), 551 ) 552 wb.properties.last_modified_by = wb.properties.creator 553 wb.properties.created = now 554 wb.properties.modified = now 555 wb.properties.title = '%s WholeCellKB' % species.name 556 wb.properties.subject = wb.properties.title 557 wb.properties.description = wb.properties.title 558 wb.properties.keywords = 'biology, systems, database, knowledge base' 559 wb.properties.category = '' 560 wb.properties.company = 'Covert Lab, Department of Bioengineering, Stanford University' 561 wb.properties.excel_base_date = CALENDAR_WINDOWS_1900 562 563 #print table of contents 564 ws = wb.create_sheet(title = 'Contents') 565 ws.default_column_dimension.auto_size = True 566 ws.default_row_dimension.height = 15 567 568 headers = ( 569 (wb.properties.title, 18, True), 570 ('Generated by WholeCellKB on %s at %s' % ( 571 now.strftime("%Y-%m-%d"), 572 settings.ROOT_URL + reverse('public.views.exportData', kwargs={'species_wid': species.wid}), 573 ), 10, False), 574 ('%s %s %s' % (html_to_ascii('©'), now.year, wb.properties.company), 10, False), 575 ) 576 for i in range(len(headers)): 577 ws.merge_cells(start_row=i, start_column=0, end_row=i, end_column=1) 578 cell = ws.cell(row=i, column=0) 579 cell.set_value_explicit(value = headers[i][0], data_type = Cell.TYPE_STRING) 580 cell.style.number_format.format_code = NumberFormat.FORMAT_TEXT 581 cell.style.font.name = 'Arial' 582 cell.style.font.size = headers[i][1] 583 cell.style.font.bold = headers[i][2] 584 cell.style.alignment.wrap_text = True 585 cell.style.alignment.horizontal = Alignment.HORIZONTAL_CENTER 586 cell.style.fill.fill_type = 'solid' 587 cell.style.fill.start_color.index = Colors.GREY1 588 589 ws.row_dimensions[i + 1].height = headers[i][1] + 7 590 591 cell = ws.cell(row=len(headers) + 1, column=0) 592 cell.set_value_explicit(value = 'Table', data_type = Cell.TYPE_STRING) 593 cell.style.number_format.format_code = NumberFormat.FORMAT_TEXT 594 cell.style.font.name = 'Arial' 595 cell.style.font.size = 10 596 cell.style.font.bold = True 597 cell.style.alignment.wrap_text = True 598 cell.style.alignment.horizontal = Alignment.HORIZONTAL_RIGHT 599 cell.style.borders.bottom.color.index = Colors.BLACK 600 cell.style.borders.bottom.border_style = Border.BORDER_THIN 601 602 cell = ws.cell(row=len(headers) + 1, column=1) 603 cell.set_value_explicit(value = 'Title', data_type = Cell.TYPE_STRING) 604 cell.style.number_format.format_code = NumberFormat.FORMAT_TEXT 605 cell.style.font.name = 'Arial' 606 cell.style.font.size = 10 607 cell.style.font.bold = True 608 cell.style.alignment.wrap_text = True 609 cell.style.alignment.horizontal = Alignment.HORIZONTAL_LEFT 610 cell.style.borders.bottom.color.index = Colors.BLACK 611 cell.style.borders.bottom.border_style = Border.BORDER_THIN 612 613 ws.freeze_panes = str('A%s' % (len(headers) + 3)) 614 ws.column_dimensions['A'].width = 8 615 ws.column_dimensions['B'].width = 75 616 617 for i in range(len(model_types)): 618 iRow = i + len(headers) + 2 619 model = modelDict[model_types[i]] 620 621 cell = ws.cell(row=iRow, column=0) 622 cell.set_value_explicit(value = '%d.' % (i+1), data_type = Cell.TYPE_STRING) 623 cell.style.number_format.format_code = NumberFormat.FORMAT_TEXT 624 cell.style.font.name = 'Arial' 625 cell.style.font.size = 10 626 cell.style.alignment.wrap_text = True 627 cell.style.alignment.horizontal = Alignment.HORIZONTAL_RIGHT 628 cell.hyperlink = '#\'%s\'!%s' % (model._meta.verbose_name_plural[:31], 'A1') 629 630 cell = ws.cell(row=iRow, column=1) 631 cell.set_value_explicit(value = model._meta.verbose_name_plural, data_type = Cell.TYPE_STRING) 632 cell.style.number_format.format_code = NumberFormat.FORMAT_TEXT 633 cell.style.font.name = 'Arial' 634 cell.style.font.size = 10 635 cell.style.alignment.wrap_text = True 636 cell.style.alignment.horizontal = Alignment.HORIZONTAL_LEFT 637 cell.hyperlink = '#\'%s\'!%s' % (model._meta.verbose_name_plural[:31], 'A1') 638 639 #print kbojects to worksheets, 1 for each model 640 for model_type in model_types: 641 model = modelDict[model_type] 642 fields = getModelDataFields(model, metadata=not is_user_anonymous) 643 644 iCol = -1 645 iCol_id = None 646 iCol_wid = None 647 for header, subheaders in get_excel_headers(model, is_user_anonymous=is_user_anonymous): 648 if header == 'ID': 649 iCol_id = iCol + 1 650 elif header == 'WID': 651 iCol_wid = iCol + 1 652 iCol += max(1, len(subheaders)) 653 654 #create worksheet 655 ws = wb.create_sheet(title = model._meta.verbose_name_plural[:31]) 656 ws.freeze_panes = str(get_column_letter(iCol_wid + 1) + '3') 657 658 #header 659 iRow1 = 0 660 iRow2 = 1 661 iCol = -1 662 for header, subheaders in get_excel_headers(model, is_user_anonymous=is_user_anonymous): 663 if len(subheaders) <= 1: 664 iCol += 1 665 cell = ws.cell(row=iRow2, column=iCol) 666 cell.set_value_explicit(value = header, data_type = Cell.TYPE_STRING) 667 else: 668 ws.merge_cells(start_row=iRow1, start_column=iCol+1, end_row=iRow1, end_column=iCol+len(subheaders)) 669 cell = ws.cell(row=iRow1, column=iCol+1) 670 cell.set_value_explicit(value = header, data_type = Cell.TYPE_STRING) 671 for subheader in subheaders: 672 iCol += 1 673 cell = ws.cell(row=iRow2, column=iCol) 674 cell.set_value_explicit(value = subheader, data_type = Cell.TYPE_STRING) 675 676 #data 677 iRow = 1 678 for obj in objectDict[model_type]: 679 iRow += 1 680 iCol = -1 681 for field in fields: 682 value, data_type, format_code = format_value_excel(obj, field) 683 try: 684 delattr(obj, "_" + field.name + "_cache") 685 except Exception, e: 686 pass 687 688 if not isinstance(value, list): 689 value = [value,] 690 data_type = [data_type, ] 691 format_code = [format_code, ] 692 693 for iter in range(len(value)): 694 iCol += 1 695 cell = ws.cell(row=iRow, column=iCol) 696 if value[iter] is None: 697 cell.set_value_explicit(b'', data_type = Cell.TYPE_NULL) 698 else: 699 cell.set_value_explicit(value = value[iter], data_type = data_type[iter]) 700 cell.style.number_format.format_code = format_code[iter] 701 cell.style.font.name = 'Arial' 702 cell.style.font.size = 10 703 cell.style.alignment.wrap_text = True 704 cell.style.fill.fill_type = 'solid' 705 if iRow % 2 == 0: 706 cell.style.fill.start_color.index = Colors.GREY1 707 else: 708 cell.style.fill.start_color.index = Colors.GREY2 709 710 #style 711 nCols = ws.get_highest_column() 712 nRows = ws.get_highest_row() 713 for iRow in range(nRows): 714 cell = ws.cell(row=iRow, column=0) 715 cell.style.font.bold = True 716 if iRow > 1: 717 if iRow % 2 == 0: 718 cell.style.fill.start_color.index = Colors.GREY3 719 else: 720 cell.style.fill.start_color.index = Colors.GREY4 721 if iRow > 0: 722 cell.style.borders.left.color.index = Colors.BLUE 723 cell.style.borders.left.border_style = Border.BORDER_THIN 724 cell.style.borders.right.color.index = Colors.BLUE 725 cell.style.borders.right.border_style = Border.BORDER_THIN 726 727 cell = ws.cell(row=iRow, column=nCols-1) 728 cell.style.borders.right.color.index = Colors.BLUE 729 cell.style.borders.right.border_style = Border.BORDER_THIN 730 731 ws.row_dimensions[iRow + 1].height = 15 732 for iCol in range(nCols): 733 cell = ws.cell(row=0, column=iCol) 734 if cell.value is not None: 735 cell.style.number_format.format_code = NumberFormat.FORMAT_TEXT 736 cell.style.font.name = 'Arial' 737 cell.style.font.size = 10 738 cell.style.font.color.index = Colors.WHITE 739 cell.style.font.bold = True 740 cell.style.alignment.horizontal = Alignment.HORIZONTAL_CENTER 741 cell.style.alignment.wrap_text = True 742 cell.style.fill.fill_type = 'solid' 743 cell.style.fill.start_color.index = Colors.BLUE 744 cell.style.borders.top.color.index = Colors.BLUE 745 cell.style.borders.top.border_style = Border.BORDER_THIN 746 if (iCol == 0) or (ws.cell(row=0, column=iCol-1).value is None): 747 cell.style.borders.left.color.index = Colors.BLUE 748 cell.style.borders.left.border_style = Border.BORDER_THIN 749 if (iCol == nCols - 1) or (ws.cell(row=0, column=iCol+1).value is None): 750 cell.style.borders.right.color.index = Colors.BLUE 751 cell.style.borders.right.border_style = Border.BORDER_THIN 752 else: 753 cell.style.borders.bottom.color.index = Colors.BLUE 754 cell.style.borders.bottom.border_style = Border.BORDER_THIN 755 756 cell = ws.cell(row=1, column=iCol) 757 cell.style.number_format.format_code = NumberFormat.FORMAT_TEXT 758 cell.style.font.name = 'Arial' 759 cell.style.font.size = 10 760 cell.style.font.color.index = Colors.WHITE 761 cell.style.font.bold = True 762 cell.style.alignment.wrap_text = True 763 cell.style.fill.fill_type = 'solid' 764 cell.style.fill.start_color.index = Colors.BLUE 765 cell.style.borders.bottom.color.index = Colors.BLUE 766 cell.style.borders.bottom.border_style = Border.BORDER_THIN 767 768 cell = ws.cell(row=nRows-1, column=iCol) 769 cell.style.borders.bottom.color.index = Colors.BLUE 770 cell.style.borders.bottom.border_style = Border.BORDER_THIN 771 772 return wb

773

774 -def get_excel_headers(model, is_user_anonymous=False):

775 fields = getModelDataFields(model, metadata=not is_user_anonymous) 776 headers = [] 777 for field in fields: 778 subheaders = [] 779 if (field.rel is not None) and isinstance(field, ForeignKey) and (not issubclass(field.rel.to, (Entry, User, ))): 780 for subfield in field.rel.to._meta.fields + field.rel.to._meta.many_to_many: 781 if not subfield.auto_created: 782 subheaders.append(html_to_ascii(subfield.verbose_name)) 783 headers.append((html_to_ascii(field.verbose_name), subheaders, )) 784 return headers

785

786 -def format_value_excel(obj, field, depth = 0):

787 value = None 788 if hasattr(obj, field.name): 789 value = getattr(obj, field.name) 790 if field.__class__ in [AutoField, BigIntegerField, IntegerField, PositiveIntegerField, PositiveSmallIntegerField, SmallIntegerField]: 791 data_type = Cell.TYPE_NUMERIC 792 format_code = NumberFormat.FORMAT_NUMBER 793 elif field.__class__ in [BooleanField, NullBooleanField]: 794 data_type = Cell.TYPE_BOOL 795 format_code = NumberFormat.FORMAT_NUMBER 796 elif field.__class__ is DecimalField: 797 data_type = Cell.TYPE_NUMERIC 798 format_code = '0.' + '0' * field.decimal_places 799 elif field.__class__ is FloatField: 800 data_type = Cell.TYPE_NUMERIC 801 format_code = NumberFormat.FORMAT_NUMBER_00 802 elif field.__class__ in [CharField, CommaSeparatedIntegerField, EmailField, FilePathField, GenericIPAddressField, IPAddressField, SlugField, URLField]: 803 data_type = Cell.TYPE_STRING 804 format_code = NumberFormat.FORMAT_TEXT 805 elif field.__class__ is TextField: 806 value = value[0:65535] 807 data_type = Cell.TYPE_STRING 808 format_code = NumberFormat.FORMAT_TEXT 809 elif field.__class__ is DateField: 810 value = datetime.datetime.combine(value, datetime.time()) 811 value = SharedDate().datetime_to_julian(date=value) 812 data_type = Cell.TYPE_NUMERIC 813 format_code = NumberFormat.FORMAT_DATE_YYYYMMDD2 814 elif field.__class__ is DateTimeField: 815 value = SharedDate().datetime_to_julian(date=value) 816 data_type = Cell.TYPE_NUMERIC 817 format_code = 'yyy-mm-dd h:mm:ss' 818 elif field.__class__ is TimeField: 819 value = SharedDate().datetime_to_julian(date=value) 820 data_type = Cell.TYPE_NUMERIC 821 format_code = NumberFormat.FORMAT_DATE_TIME4 822 elif field.__class__ is ForeignKey: 823 subobj = value 824 if issubclass(field.rel.to, (Entry, User)): 825 value = None 826 if subobj is not None: 827 value = unicode(subobj) 828 data_type = Cell.TYPE_STRING 829 format_code = NumberFormat.FORMAT_TEXT 830 elif depth > 0: 831 value = {} 832 for subfield in field.rel.to._meta.fields + field.rel.to._meta.many_to_many: 833 if not subfield.auto_created and getattr(subobj, subfield.name) is not None: 834 value[subfield.name], junk1, junk2 = format_value_excel(subobj, subfield, depth = depth + 1) 835 data_type = Cell.TYPE_STRING 836 format_code = NumberFormat.FORMAT_TEXT 837 else: 838 value = [] 839 data_type = [] 840 format_code = [] 841 for subfield in field.rel.to._meta.fields + field.rel.to._meta.many_to_many: 842 if not subfield.auto_created: 843 tmp_val, tmp_data_type, tmp_format_code = format_value_excel(subobj, subfield, depth = depth + 1) 844 845 if depth == 0 and isinstance(tmp_val, (list, dict)): 846 tmp_val = simplejson.dumps(tmp_val)[1:-1] 847 848 value.append(tmp_val) 849 data_type.append(tmp_data_type) 850 format_code.append(tmp_format_code) 851 elif field.__class__ is ManyToManyField: 852 data_type = Cell.TYPE_STRING 853 format_code = NumberFormat.FORMAT_TEXT 854 855 if value is None: 856 pass 857 elif issubclass(field.rel.to, Entry): 858 subobjs = value.all() 859 value = [] 860 for subobj in subobjs: 861 value.append(unicode(subobj)) 862 if depth == 0: 863 value = ', '.join(value) 864 else: 865 subobjs = value.all() 866 value = [] 867 for subobj in subobjs: 868 subvalue = {} 869 for subfield in field.rel.to._meta.fields + field.rel.to._meta.many_to_many: 870 if not subfield.auto_created and getattr(subobj, subfield.name) is not None: 871 subvalue[subfield.name], junk1, junk2 = format_value_excel(subobj, subfield, depth = depth + 1) 872 value.append(subvalue) 873 874 if depth == 0: 875 value = simplejson.dumps(value)[1:-1] 876 else: 877 raise Exception('%s class not supported' % field.__class__.__name__) 878 879 return (value, data_type, format_code, )

880

881 -def batch_import_from_excel(species_wid, fileName, user):

882 models = get_models_in_saving_order() 883 884 #read data 885 data = read_excel_data(fileName) 886 887 #check species WID 888 if data.has_key('Species'): 889 if len(data['Species']) > 1: 890 raise ValidationError('Please define species 1 at a time.') 891 species = data['Species'][0] 892 if species.has_key('id'): 893 species_id = species['id'] 894 else: 895 species_id = None 896 species_wid = species['wid'] 897 else: 898 try: 899 species = Species.objects.get(wid=species_wid) 900 species_id = species.id 901 except ObjectDoesNotExist: 902 raise ValidationError('Please edit an editing PGDB') 903 904 if len(Species.objects.exclude(id=species_id).filter(wid=species_wid)) > 0: 905 raise ValidationError('Species WID must be unique.') 906 907 #check object WIDs unique 908 old_wids_list = [] 909 if species_id is not None: 910 qs = SpeciesComponent.objects.values('wid', 'model_type').filter(species__id=species_id) 911 for model in models: 912 if data.has_key(model.__name__): 913 for obj_data in data[model.__name__]: 914 if obj_data['id'] is not None: 915 qs = qs.exclude(id=obj_data['id']) 916 917 for x in qs: 918 old_wids_list.append(x['wid']) 919 920 new_wids_list = [] 921 for model in models: 922 if data.has_key(model.__name__): 923 for obj_data in data[model.__name__]: 924 new_wids_list.append(obj_data['wid']) 925 else: 926 data[model.__name__] = [] 927 928 if len(set(new_wids_list)) < len(new_wids_list): 929 print new_wids_list 930 raise ValidationError('WIDs must be unique') 931 if len(set(new_wids_list) & set(old_wids_list)) > 0: 932 raise ValidationError('WIDs not unique: %s' % ', '.join(set(new_wids_list) & set(old_wids_list))) 933 934 #get all data for the current species 935 all_obj_wids = {species_wid: 'Species'} 936 all_obj_data = {species_wid: species} 937 all_obj_data_by_model = {'Species': [species]} 938 for model_name, model in getModels().iteritems(): 939 if not issubclass(model, SpeciesComponent): 940 continue 941 all_obj_data_by_model[model_name] = [] 942 for obj in data[model_name]: 943 all_obj_wids[obj['wid']] = obj['model_type'] 944 all_obj_data[obj['wid']] = obj 945 all_obj_data_by_model[model_name].append(obj) 946 947 if species_id is not None: 948 qs = model.objects.select_related().filter(species__id=species_id) 949 for obj in data[model_name]: 950 if obj_data['id'] is not None: 951 qs = qs.exclude(id=obj['id']) 952 for obj in qs: 953 all_obj_wids[obj.wid] = obj.model_type 954 all_obj_data[obj.wid] = obj 955 all_obj_data_by_model[model_name].append(obj) 956 957 ''' validation ''' 958 errors = [] 959 960 #fields 961 for model in models: 962 for obj_data in data[model.__name__]: 963 try: 964 obj_data = validate_object_fields(model, obj_data, all_obj_wids, species_wid, obj_data['wid']) 965 except ValidationError as error: 966 errors.append('%s %s invalid: %s' % (model.__name__, obj_data['wid'], format_error_as_html_list(error))) 967 if len(errors) > 0: 968 raise ValidationError(format_list_html(errors)) 969 970 #objects 971 errors = [] 972 for model in models: 973 for obj_data in data[model.__name__]: 974 try: 975 validate_model_objects(model, 976 obj_data, 977 all_obj_data=all_obj_data, 978 all_obj_data_by_model=all_obj_data_by_model) 979 except ValidationError as error: 980 errors.append('%s %s invalid: %s' % (model.__name__, obj_data['wid'], format_error_as_html_list(error))) 981 if len(errors) > 0: 982 raise ValidationError(format_list_html(errors)) 983 984 #uniqueness 985 for model in models: 986 try: 987 validate_model_unique(model, 988 all_obj_data_by_model[model.__name__], 989 all_obj_data=all_obj_data, 990 all_obj_data_by_model=all_obj_data_by_model) 991 except ValidationError as error: 992 errors.append('%s invalid: %s' % (model.__name__, format_error_as_html_list(error))) 993 if len(errors) > 0: 994 raise ValidationError(format_list_html(errors)) 995 996 ''' save ''' 997 #commit 998 obj_list = {} 999 for model in models: 1000 for obj_data in data[model.__name__]: 1001 if obj_data['id'] is None: 1002 obj = model() 1003 else: 1004 obj = model.objects.get(id = obj_data['id']) 1005 obj_list[obj_data['wid']] = save_object_data(species_wid, obj, obj_data, obj_list, user, save=False, save_m2m=False) 1006 1007 #base 1008 for model in models: 1009 for obj_data in data[model.__name__]: 1010 obj = obj_list[obj_data['wid']] 1011 obj_list[obj_data['wid']] = save_object_data(species_wid, obj, obj_data, obj_list, user, save=True, save_m2m=False) 1012 1013 #relations 1014 for model in models: 1015 for obj_data in data[model.__name__]: 1016 obj = obj_list[obj_data['wid']] 1017 obj_list[obj_data['wid']] = save_object_data(species_wid, obj, obj_data, obj_list, user, save=True, save_m2m=True)

1018

1019 -def read_excel_data(filename):

1020 #read workbook 1021 wb = load_workbook(filename = filename) 1022 1023 #get models 1024 models = getModels() 1025 1026 #initialize errors 1027 errors = [] 1028 1029 #check headers 1030 for modelname, model in models.iteritems(): 1031 ws = wb.get_sheet_by_name(name = model._meta.verbose_name_plural[:31]) 1032 if ws is None: 1033 continue 1034 1035 iRow1 = 0 1036 iRow2 = 1 1037 iCol = -1 1038 for header, subheaders in get_excel_headers(model): 1039 if len(subheaders) <= 1: 1040 iCol += 1 1041 cell = ws.cell(row=iRow2, column=iCol) 1042 if cell.value != header: 1043 errors.append('Expecting worksheet "%s" header %s%s "%s", got "%s"' % (ws.title, get_column_letter(iCol+1), iRow2+1, header, cell.value)) 1044 else: 1045 cell = ws.cell(row=iRow1, column=iCol+1) 1046 if cell.value != header: 1047 errors.append('Expecting worksheet "%s" header %s%s "%s", got "%s"' % (ws.title, get_column_letter(iCol+2), iRow1+1, header, cell.value)) 1048 for subheader in subheaders: 1049 iCol += 1 1050 cell = ws.cell(row=iRow2, column=iCol) 1051 if cell.value != subheader: 1052 errors.append('Expecting worksheet "%s" header %s%s "%s", got "%s"' % (ws.title, get_column_letter(iCol+1), iRow2+1, subheader, cell.value)) 1053 1054 if (iCol + 1) != ws.get_highest_column(): 1055 errors.append('Expecting worksheet "%s" to have exactly %s columns' % (ws.title, iCol + 1)) 1056 1057 if len(errors) > 0: 1058 raise ValidationError(format_list_html(errors)) 1059 1060 #get data 1061 newobjects = {} 1062 for modelname, model in models.iteritems(): 1063 ws = wb.get_sheet_by_name(name = model._meta.verbose_name_plural[:31]) 1064 if ws is None: 1065 continue 1066 1067 fields = getModelDataFields(model) 1068 1069 iCol = -1 1070 iCol_id = None 1071 for header, subheaders in get_excel_headers(model): 1072 if header == 'ID': 1073 iCol_id = iCol + 1 1074 iCol += max(1, len(subheaders)) 1075 1076 #data 1077 newobjects[modelname] = [] 1078 for iRow in range(2, ws.get_highest_row()): 1079 id = ws.cell(row=iRow, column=iCol_id).value 1080 if id == '': 1081 id is None 1082 obj = {'id': id, 'model_type': modelname} 1083 1084 iField = -1 1085 iCol = -1 1086 for header, subheaders in get_excel_headers(model): 1087 iField += 1 1088 iCol += max(1, len(subheaders)) 1089 field = fields[iField] 1090 1091 if not field.editable or field.auto_created: 1092 continue 1093 1094 if isinstance(field, (CharField, SlugField, TextField)): 1095 value = ws.cell(row=iRow, column=iCol).value 1096 if value is None: 1097 value = '' 1098 obj[field.name] = value 1099 elif isinstance(field, (BooleanField, FloatField, IntegerField, NullBooleanField, PositiveIntegerField)): 1100 value = ws.cell(row=iRow, column=iCol).value 1101 if value == '': 1102 value = None 1103 obj[field.name] = value 1104 elif isinstance(field, ForeignKey): 1105 if issubclass(field.rel.to, Entry): 1106 value = ws.cell(row=iRow, column=iCol).value 1107 if value == '': 1108 value = None 1109 obj[field.name] = value 1110 else: 1111 obj[field.name] = {} 1112 iCol2 = iCol - max(1, len(subheaders)) 1113 is_blank = True 1114 for subfield in field.rel.to._meta.fields + field.rel.to._meta.many_to_many: 1115 if subfield.auto_created: 1116 continue 1117 1118 iCol2 += 1 1119 if isinstance(subfield, (CharField, SlugField, TextField)): 1120 value = ws.cell(row=iRow, column=iCol2).value 1121 if value is None: 1122 value = '' 1123 elif isinstance(subfield, (BooleanField, FloatField, IntegerField, NullBooleanField, PositiveIntegerField)): 1124 value = ws.cell(row=iRow, column=iCol2).value 1125 if value == '': 1126 value = None 1127 elif isinstance(subfield, ForeignKey) and issubclass(subfield.rel.to, Entry): 1128 value = ws.cell(row=iRow, column=iCol2).value 1129 if value == '': 1130 value = None 1131 elif isinstance(subfield, ManyToManyField) and issubclass(subfield.rel.to, Entry): 1132 value = ws.cell(row=iRow, column=iCol2).value 1133 if value is None or value == '': 1134 value = [] 1135 else: 1136 value = value.split(', ') 1137 elif isinstance(subfield, ForeignKey): 1138 value = ws.cell(row=iRow, column=iCol2).value 1139 if value == '': 1140 value = None 1141 else: 1142 value = simplejson.loads(value) 1143 elif isinstance(subfield, ManyToManyField): 1144 value = ws.cell(row=iRow, column=iCol2).value 1145 if value is None or value == '': 1146 value = [] 1147 else: 1148 value = simplejson.loads('[' + value + ']') 1149 1150 obj[field.name][subfield.name] = value 1151 1152 if value is not None and not (isinstance(value, (str, unicode)) and value == '') and not (isinstance(value, list) and len(value) == 0): 1153 is_blank = False 1154 1155 if is_blank: 1156 obj[field.name] = None 1157 elif isinstance(field, ManyToManyField): 1158 if issubclass(field.rel.to, Entry): 1159 value = ws.cell(row=iRow, column=iCol).value 1160 if value is None or value == '': 1161 obj[field.name] = [] 1162 else: 1163 obj[field.name] = value.split(', ') 1164 else: 1165 value = ws.cell(row=iRow, column=iCol).value 1166 if value is None or value == '': 1167 obj[field.name] = [] 1168 else: 1169 try: 1170 obj[field.name] = simplejson.loads('[' + value + ']') 1171 except: 1172 errors.append('Invalid json at field %s of %s %s at cell %s%s' % (field.name, modelname, obj['wid'], get_column_letter(iCol+1), iRow+1, )) 1173 1174 newobjects[modelname].append(obj) 1175 1176 if len(errors) > 0: 1177 raise ValidationError(format_list_html(errors)) 1178 1179 return newobjects

1180

1181 -def format_error_as_html_list(error):

1182 fullmsg = '<ul>' 1183 for key, msg in error.message_dict.iteritems(): 1184 if isinstance(msg, (str, unicode, )): 1185 msg = [msg] 1186 1187 if key == '__all__': 1188 fullmsg += '<li>%s</li>' % (' '.join(msg), ) 1189 else: 1190 fullmsg += '<li>%s: %s</li>' % (key, ' '.join(msg), ) 1191 fullmsg += '</ul>' 1192 return fullmsg

1193

1194 -def remove_new_objects(newobjects, newsubobjects):

1195 for model in get_models_in_saving_order(): 1196 for obj in newobjects[model.__name__]: 1197 try: 1198 obj.delete() 1199 except: 1200 pass 1201 for obj in newsubobjects: 1202 if obj.pk is not None: 1203 try: 1204 obj.delete() 1205 except: 1206 pass

1207

1208 -def objectToQuerySet(obj, model = None):

1209 if model is None: 1210 model = obj.__class__ 1211 qs = model.objects.none() 1212 qs._result_cache.append(obj) 1213 return qs

1214

1215 -def format_field_detail_view(obj, field_name, is_user_anonymous):

1216 if hasattr(obj, 'get_as_html_%s' % field_name): 1217 val = getattr(obj, 'get_as_html_%s' % field_name)(is_user_anonymous) 1218 if isinstance(val, float) and val != 0. and val is not None: 1219 return ('%.' + str(int(math.ceil(max(0, -math.log10(abs(val)))+2))) + 'f') % val 1220 return val 1221 1222 field = obj.__class__._meta.get_field_by_name(field_name)[0] 1223 value = getattr(obj, field_name) 1224 1225 if isinstance(field, RelatedObject): 1226 field_model = field.model 1227 value = value.all() 1228 elif isinstance(field, ManyToManyField): 1229 field_model = field.rel.to 1230 value = value.all() 1231 elif isinstance(field, ForeignKey): 1232 field_model = field.rel.to 1233 1234 if isinstance(field, (ManyToManyField, RelatedObject)): 1235 if issubclass(field_model, Entry): 1236 results = [] 1237 for subvalue in value: 1238 results.append('<a href="%s">%s</a>' % (subvalue.get_absolute_url(), subvalue.wid)) 1239 return format_list_html(results) 1240 1241 results = [] 1242 for subvalue in value: 1243 if issubclass(field_model, (EntryBooleanData, )): 1244 if subvalue.value: 1245 txt = 'Yes' 1246 else: 1247 txt = 'No' 1248 elif issubclass(field_model, (EntryCharData, EntryFloatData, EntryPositiveFloatData, EntryTextData, )): 1249 if subvalue.units is None: 1250 txt = '%s' % (subvalue.value, ) 1251 else: 1252 txt = '%s (%s)' % (subvalue.value, subvalue.units, ) 1253 else: 1254 subresults = [] 1255 for subfield in field_model._meta.fields + field_model._meta.many_to_many: 1256 if subfield.auto_created: 1257 pass 1258 elif isinstance(subfield, ManyToManyField) and issubclass(subfield.rel.to, Evidence): 1259 pass 1260 else: 1261 subresults.append('%s: %s' % (subfield.verbose_name, format_field_detail_view(subvalue, subfield.name, is_user_anonymous), )) 1262 txt = ', '.join(subresults) 1263 1264 if issubclass(field_model, EvidencedEntryData) and len(subvalue.evidence.all()) > 0: 1265 txt = txt + (' <a href="javascript:void(0)" onclick="toggleEvidence(this);">(Show evidence)</a><div class="evidence" style="display:none;">%s</div>' % format_evidence(subvalue.evidence.all())) 1266 results.append('<div>%s</div>' % txt) 1267 return format_list_html(results) 1268 1269 elif isinstance(field, ForeignKey): 1270 if issubclass(field_model, Entry): 1271 if value is not None: 1272 return '<a href="%s">%s</a>' % (value.get_absolute_url(), value.wid) 1273 else: 1274 return '' 1275 1276 if value is None: 1277 return '' 1278 1279 if issubclass(field_model, (EntryBooleanData, )): 1280 if value.value: 1281 txt = 'Yes' 1282 else: 1283 txt = 'No' 1284 elif issubclass(field_model, (EntryCharData, EntryFloatData, EntryPositiveFloatData, EntryTextData, )): 1285 if value.units is None: 1286 txt = '%s' % (value.value, ) 1287 else: 1288 txt = '%s (%s)' % (value.value, value.units, ) 1289 else: 1290 results = [] 1291 for subfield in field_model._meta.fields + field_model._meta.many_to_many: 1292 if subfield.auto_created: 1293 pass 1294 elif isinstance(subfield, ManyToManyField) and issubclass(subfield.rel.to, Evidence): 1295 pass 1296 else: 1297 results.append('%s: %s' % (subfield.verbose_name, format_field_detail_view(value, subfield.name, is_user_anonymous), )) 1298 txt = ', '.join(results) 1299 1300 if issubclass(field_model, EvidencedEntryData) and len(value.evidence.all()) > 0: 1301 txt = txt + (' <a href="javascript:void(0)" onclick="toggleEvidence(this);">(Show evidence)</a><div class="evidence" style="display:none;">%s</div>' % format_evidence(value.evidence.all())) 1302 1303 return '<div>%s</div>' % txt 1304 1305 elif (field.choices is not None) and (len(field.choices) > 0) and (not isinstance(field, (BooleanField, NullBooleanField))): 1306 choices = [x[0] for x in field.choices] 1307 if value in choices: 1308 value = field.choices[choices.index(value)][1] 1309 return value 1310 1311 elif isinstance(value, float) and value != 0 and value is not None: 1312 return ('%.' + str(int(math.ceil(max(0, -math.log10(abs(value)))+2))) + 'f') % value 1313 1314 else: 1315 return value 1316 1317 return ''

1318

1319 -def format_field_detail_view_helper():

1320 pass

1321

1322 -def convert_modelobject_to_stdobject(obj, is_user_anonymous=False, ancestors = []):

1323 model = obj.__class__ 1324 1325 if issubclass(model, Entry) and len(ancestors) > 0: 1326 return obj.wid 1327 1328 objDict = {} 1329 if issubclass(model, Entry): 1330 fields = getModelDataFields(model, metadata=not is_user_anonymous) 1331 else: 1332 fields = model._meta.fields + model._meta.many_to_many 1333 1334 for field in fields: 1335 model_val = getattr(obj, field.name) 1336 if field.auto_created: 1337 continue 1338 1339 if isinstance(field, ForeignKey): 1340 if model_val is None: 1341 stdobj_val = None 1342 else: 1343 if issubclass(field.rel.to, Entry): 1344 stdobj_val = model_val.wid 1345 elif issubclass(field.rel.to, User): 1346 stdobj_val = model_val.username 1347 else: 1348 stdobj_val = convert_modelobject_to_stdobject(model_val, is_user_anonymous, ancestors + [obj]) 1349 elif isinstance(field, ManyToManyField): 1350 stdobj_val = [] 1351 if issubclass(field.rel.to, Entry): 1352 for model_subval in model_val.all(): 1353 stdobj_val.append(model_subval.wid) 1354 elif issubclass(field.rel.to, User): 1355 for model_subval in model_val.all(): 1356 stdobj_val.append(model_subval.username) 1357 else: 1358 for model_subval in model_val.all(): 1359 stdobj_subval = convert_modelobject_to_stdobject(model_subval, is_user_anonymous, ancestors + [obj]) 1360 stdobj_val.append(stdobj_subval) 1361 elif field.choices is not None and len(field.choices) > 0: 1362 choices = [x[0] for x in field.choices] 1363 if model_val in choices: 1364 stdobj_val = unicode(field.choices[choices.index(model_val)][1]) 1365 else: 1366 stdobj_val = unicode(model_val) 1367 else: 1368 stdobj_val = unicode(model_val) 1369 1370 objDict[field.name] = stdobj_val 1371 1372 for field in model._meta.get_all_related_objects() + model._meta.get_all_related_many_to_many_objects(): 1373 if field.get_accessor_name() == '+': 1374 continue 1375 1376 objDict[field.get_accessor_name()] = [] 1377 for model_subval in getattr(obj, field.get_accessor_name()).all(): 1378 if model_subval not in ancestors: 1379 objDict[field.get_accessor_name()].append(convert_modelobject_to_stdobject(model_subval, is_user_anonymous, ancestors + [obj])) 1380 1381 return objDict

1382

1383 -def convert_modelobject_to_xml(obj, xml_doc, is_user_anonymous, ancestors = []):

1384 model = obj.__class__ 1385 xml_obj = xml_doc.createElement('object') 1386 1387 if issubclass(model, Entry) and len(ancestors) > 0: 1388 xml_obj.appendChild(xml_doc.createTextNode(unicode(obj.wid))) 1389 return xml_obj 1390 1391 xml_field = xml_doc.createElement('field') 1392 xml_field.setAttribute('name', 'model') 1393 xml_field.appendChild(xml_doc.createTextNode(unicode(model.__name__))) 1394 xml_obj.appendChild(xml_field) 1395 1396 if issubclass(model, Entry): 1397 fields = getModelDataFields(model, metadata=not is_user_anonymous) 1398 else: 1399 fields = model._meta.fields + model._meta.many_to_many 1400 1401 for field in fields: 1402 model_val = getattr(obj, field.name) 1403 if field.auto_created: 1404 continue 1405 1406 xml_field = xml_doc.createElement('field') 1407 xml_field.setAttribute('name', field.name) 1408 xml_obj.appendChild(xml_field) 1409 1410 if isinstance(field, ForeignKey): 1411 if model_val is None: 1412 pass 1413 else: 1414 if issubclass(field.rel.to, Entry): 1415 xml_field.appendChild(xml_doc.createTextNode(unicode(model_val.wid))) 1416 elif issubclass(field.rel.to, User): 1417 xml_field.appendChild(xml_doc.createTextNode(unicode(model_val.username))) 1418 else: 1419 xml_field.appendChild(convert_modelobject_to_xml(model_val, xml_doc, is_user_anonymous, ancestors + [obj])) 1420 elif isinstance(field, ManyToManyField): 1421 if issubclass(field.rel.to, Entry): 1422 for model_subval in model_val.all(): 1423 doc = xml_doc.createElement('object') 1424 doc.appendChild(xml_doc.createTextNode(unicode(model_subval.wid))) 1425 xml_field.appendChild(doc) 1426 elif issubclass(field.rel.to, User): 1427 for model_subval in model_val.all(): 1428 doc = xml_doc.createElement('object') 1429 doc.appendChild(xml_doc.createTextNode(unicode(model_subval.username))) 1430 xml_field.appendChild(doc) 1431 else: 1432 for model_subval in model_val.all(): 1433 xml_field.appendChild(convert_modelobject_to_xml(model_subval, xml_doc, is_user_anonymous, ancestors + [obj])) 1434 elif field.choices is not None and len(field.choices) > 0: 1435 choices = [x[0] for x in field.choices] 1436 if model_val in choices: 1437 xml_field.appendChild(xml_doc.createTextNode(unicode(field.choices[choices.index(model_val)][1]))) 1438 else: 1439 xml_field.appendChild(xml_doc.createTextNode(unicode(model_val))) 1440 else: 1441 xml_field.appendChild(xml_doc.createTextNode(unicode(model_val))) 1442 1443 for field in model._meta.get_all_related_objects() + model._meta.get_all_related_many_to_many_objects(): 1444 if field.get_accessor_name() == '+': 1445 continue 1446 1447 xml_field = xml_doc.createElement('field') 1448 xml_field.setAttribute('name', field.get_accessor_name()) 1449 xml_obj.appendChild(xml_field) 1450 1451 for model_subval in getattr(obj, field.get_accessor_name()).all(): 1452 if model_subval not in ancestors: 1453 xml_field.appendChild(convert_modelobject_to_xml(model_subval, xml_doc, is_user_anonymous, ancestors + [obj])) 1454 1455 return xml_obj

1456

1457 -def get_edit_form_fields(species_wid, model, obj=None):

1458 model_type = model.__name__ 1459 form_fields = [] 1460 initial_values = {} 1461 1462 if issubclass(model, Entry): 1463 fields = [model._meta.get_field_by_name(x)[0] for x in model._meta.field_list] 1464 else: 1465 fields = model._meta.fields + model._meta.many_to_many 1466 1467 for model_field in fields: 1468 if not model_field.editable or model_field.auto_created: 1469 continue 1470 1471 type = None 1472 value = None 1473 fields = None 1474 choices = None 1475 1476 if isinstance(model_field, ForeignKey): 1477 if issubclass(model_field.rel.to, Entry): 1478 type = 'select' 1479 if issubclass(model_field.rel.to, SpeciesComponent): 1480 choices = tuple([(x.wid, x.wid) for x in model_field.rel.to.objects.filter(species__wid=species_wid)]) 1481 else: 1482 choices = tuple([(x.wid, x.wid) for x in model_field.rel.to.objects.all()]) 1483 if obj is not None and getattr(obj, model_field.name) is not None: 1484 value = getattr(obj, model_field.name).wid 1485 else: 1486 type = 'ForeignKey' 1487 if obj is not None: 1488 subobj = getattr(obj, model_field.name) 1489 else: 1490 subobj = None 1491 fields, value = get_edit_form_fields(species_wid, model_field.rel.to, obj=subobj) 1492 if obj is not None and subobj is None: 1493 value = None 1494 elif isinstance(model_field, ManyToManyField): 1495 if issubclass(model_field.rel.to, Entry): 1496 type = 'multiselect' 1497 if issubclass(model_field.rel.to, SpeciesComponent): 1498 choices = tuple([(x.wid, x.wid) for x in model_field.rel.to.objects.filter(species__wid=species_wid)]) 1499 else: 1500 choices = tuple([(x.wid, x.wid) for x in model_field.rel.to.objects.all()]) 1501 if obj is not None: 1502 value = [x.wid for x in getattr(obj, model_field.name).all()] 1503 else: 1504 value = [] 1505 else: 1506 type = 'ManyToManyField' 1507 fields, tmp = get_edit_form_fields(species_wid, model_field.rel.to) 1508 value = [] 1509 if obj is not None: 1510 for subobj in getattr(obj, model_field.name).all(): 1511 junk, tmp = get_edit_form_fields(species_wid, model_field.rel.to, subobj) 1512 value.append(tmp) 1513 elif isinstance(model_field, TextField): 1514 type = 'textarea' 1515 if obj is not None: 1516 value = getattr(obj, model_field.name) 1517 elif isinstance(model_field, (BooleanField, NullBooleanField, )): 1518 type = 'select' 1519 choices = [ 1520 (True, 'True'), 1521 (False, 'False'), 1522 ] 1523 if obj is not None: 1524 value = getattr(obj, model_field.name) 1525 elif model_field.choices is not None and len(model_field.choices) >= 1: 1526 type = 'select' 1527 choices = model_field.choices 1528 if obj is not None: 1529 value = getattr(obj, model_field.name) 1530 else: 1531 type = 'text' 1532 if obj is not None: 1533 value = getattr(obj, model_field.name) 1534 1535 form_fields.append({ 1536 'type': type, 1537 'choices': choices, 1538 'fields': fields, 1539 'verbose_name': model_field.verbose_name, 1540 'name': model_field.name}) 1541 1542 initial_values[model_field.name] = value 1543 1544 return (form_fields, initial_values)

1545

1546 -def get_edit_form_data(model, POST, field_prefix=''):

1547 #get fields 1548 if issubclass(model, Entry): 1549 fields = [model._meta.get_field_by_name(x)[0] for x in model._meta.field_list] 1550 else: 1551 fields = model._meta.fields + model._meta.many_to_many 1552 1553 #process properties 1554 data = {} 1555 is_blank = True 1556 for field in fields: 1557 if not field.editable or field.auto_created: 1558 continue 1559 1560 if isinstance(field, ForeignKey): 1561 if issubclass(field.rel.to, Entry): 1562 val = POST.get('%s%s' % (field_prefix, field.name, ), None) 1563 if val == '': 1564 val is None 1565 else: 1566 val = get_edit_form_data(field.rel.to, POST, field_prefix='%s%s_' % (field_prefix, field.name, )) 1567 elif isinstance(field, ManyToManyField): 1568 val = [] 1569 if issubclass(field.rel.to, Entry): 1570 wids = POST.getlist('%s%s' % (field_prefix, field.name, ), []) 1571 for wid in wids: 1572 if wid is not None and wid != '': 1573 val.append(wid) 1574 else: 1575 keys = POST.dict().keys() 1576 idxs = [] 1577 for key in keys: 1578 match = re.match(r'^%s%s_(\d+)_.+' % (field_prefix, field.name, ), key) 1579 if match: 1580 idxs.append(match.group(1)) 1581 idxs = set(idxs) 1582 1583 for idx in idxs: 1584 tmp = get_edit_form_data(field.rel.to, POST, field_prefix='%s%s_%s_' % (field_prefix, field.name, idx, )) 1585 if tmp is not None: 1586 val.append(tmp) 1587 else: 1588 val = POST.get('%s%s' % (field_prefix, field.name, ), None) 1589 if field.null is True and val == '': 1590 val = None 1591 1592 data[field.name] = val 1593 if not (val is None or (isinstance(val, (str, unicode)) and val == '') or (isinstance(val, list) and len(val) == 0)): 1594 is_blank = False 1595 1596 #return 1597 if is_blank: 1598 return None 1599 return data

1600

1601 -def validate_object_fields(model, data, wids, species_wid, entry_wid):

1602 if issubclass(model, Entry): 1603 fields = [model._meta.get_field_by_name(x)[0] for x in model._meta.field_list] 1604 else: 1605 fields = model._meta.fields + model._meta.many_to_many 1606 1607 if issubclass(model, Entry): 1608 data['species'] = species_wid 1609 1610 errors = {} 1611 for field in fields: 1612 if not field.editable or field.auto_created: 1613 continue 1614 1615 try: 1616 if isinstance(field, ForeignKey): 1617 if issubclass(model, Evidence) and issubclass(field.rel.to, SpeciesComponent): 1618 data[field.name] = entry_wid 1619 1620 if not data.has_key(field.name): 1621 data[field.name] = None 1622 1623 if data[field.name] is None: 1624 if not field.null: 1625 raise ValidationError('Undefined WID %s' % data[field.name]) 1626 else: 1627 if issubclass(field.rel.to, Entry): 1628 if not wids.has_key(data[field.name]): 1629 raise ValidationError('Undefined WID %s' % data[field.name]) 1630 if not issubclass(getModel(wids[data[field.name]]), field.rel.to): 1631 raise ValidationError('Invalid WID %s' % data[field.name]) 1632 else: 1633 try: 1634 data[field.name] = validate_object_fields(field.rel.to, data[field.name], wids, species_wid, entry_wid) 1635 except ValidationError as error: 1636 tmp = ['%s: %s' % (key, val, ) for key, val in error.message_dict.iteritems()] 1637 raise ValidationError('<ul><li>%s</li></ul>' % '</li><li>'.join(tmp)) 1638 1639 elif isinstance(field, ManyToManyField): 1640 if not data.has_key(field.name): 1641 data[field.name] = [] 1642 1643 if issubclass(field.rel.to, Entry): 1644 for wid in data[field.name]: 1645 if not wids.has_key(wid): 1646 raise ValidationError('Undefined WID %s' % wid) 1647 if not issubclass(getModel(wids[wid]), field.rel.to): 1648 raise ValidationError('Invalid WID %s' % wid) 1649 else: 1650 for idx in range(len(data[field.name])): 1651 try: 1652 data[field.name][idx] = validate_object_fields(field.rel.to, data[field.name][idx], wids, species_wid, entry_wid) 1653 except ValidationError as error: 1654 tmp = ['%s: %s' % (key, val, ) for key, val in error.message_dict.iteritems()] 1655 raise ValidationError('<ul><li>%s</li></ul>' % '</li><li>'.join(tmp)) 1656 else: 1657 if data.has_key(field.name): 1658 data[field.name] = field.clean(data[field.name], None) 1659 else: 1660 if field.default != NOT_PROVIDED: 1661 data[field.name] = field.clean(field.default, None) 1662 else: 1663 data[field.name] = field.clean(None, None) 1664 1665 except ValidationError as error: 1666 errors[field.name] = ', '.join(error.messages) 1667 1668 if len(errors) > 0: 1669 raise ValidationError(errors) 1670 1671 return data

1672

1673 -def validate_model_objects(model, obj_data, all_obj_data=None, all_obj_data_by_model=None):

1674 parent_list = list(model._meta.get_parent_list()) 1675 parent_list.reverse() 1676 for parent_model in parent_list + [model]: 1677 if hasattr(parent_model._meta, 'clean'): 1678 parent_model._meta.clean(parent_model, obj_data, all_obj_data=all_obj_data, all_obj_data_by_model=all_obj_data_by_model)

1679

1680 -def validate_model_unique(model, model_objects_data, all_obj_data=None, all_obj_data_by_model=None):

1681 if not issubclass(model, SpeciesComponent): 1682 return 1683 1684 parent_list = list(model._meta.get_parent_list()) 1685 parent_list.reverse() 1686 1687 if all_obj_data is None: 1688 if model_objects_data[0] is Entry: 1689 species_wid = model_objects_data[0].species.wid 1690 else: 1691 species_wid = model_objects_data[0]['species'] 1692 1693 if issubclass(model, SpeciesComponent): 1694 species_id = Species.objects.values('id').get(wid=species_wid)['id'] 1695 qs = model.objects.filter(species__id=species_id) 1696 else: 1697 qs = model.objects.all() 1698 for obj in model_objects_data: 1699 if obj['id'] is not None: 1700 qs = qs.exclude(id=obj['id']) 1701 for obj in qs: 1702 model_objects_data.append(obj) 1703 1704 for ancestor_model in parent_list + [model]: 1705 if hasattr(ancestor_model._meta, 'validate_unique'): 1706 ancestor_model._meta.validate_unique(ancestor_model, model_objects_data, 1707 all_obj_data=all_obj_data, all_obj_data_by_model=all_obj_data_by_model)

1708

1709 -def save_object_data(species_wid, obj, obj_data, obj_list, user, save=False, save_m2m=False):

1710 model = obj.__class__ 1711 if isinstance(obj, Entry) and obj.pk is None: 1712 obj.created_user = user 1713 obj.last_updated_user = user 1714 1715 if issubclass(model, Entry): 1716 fields = [model._meta.get_field_by_name(x)[0] for x in model._meta.field_list] 1717 else: 1718 fields = model._meta.fields + model._meta.many_to_many 1719 1720 #regular and foreign key fields 1721 for field in fields: 1722 if not field.editable or field.auto_created: 1723 continue 1724 1725 if isinstance(field, ForeignKey): 1726 if save: 1727 if issubclass(field.rel.to, Entry): 1728 if obj_data[field.name] is not None: 1729 if obj_list.has_key(obj_data[field.name]): 1730 tmp = obj_list[obj_data[field.name]] 1731 elif issubclass(field.rel.to, SpeciesComponent): 1732 tmp = field.rel.to.objects.get(species__wid=species_wid, wid=obj_data[field.name]) 1733 else: 1734 tmp = field.rel.to.objects.get(wid=obj_data[field.name]) 1735 setattr(obj, field.name, tmp) 1736 else: 1737 setattr(obj, field.name, None) 1738 else: 1739 if obj_data[field.name] is not None: 1740 setattr(obj, field.name, save_object_data(species_wid, field.rel.to(), obj_data[field.name], obj_list, user, save=save, save_m2m=save_m2m)) 1741 else: 1742 setattr(obj, field.name, None) 1743 elif isinstance(field, ManyToManyField): 1744 pass 1745 else: 1746 setattr(obj, field.name, obj_data[field.name]) 1747 1748 #save 1749 if save: 1750 if isinstance(obj, SpeciesComponent): 1751 obj.species = Species.objects.get(wid=species_wid) 1752 obj.full_clean() 1753 obj.save() 1754 1755 #many-to-many fields 1756 for field in fields: 1757 if not field.editable or field.auto_created: 1758 continue 1759 1760 if isinstance(field, ManyToManyField): 1761 if save_m2m: 1762 getattr(obj, field.name).clear() 1763 if issubclass(field.rel.to, Entry): 1764 for wid in obj_data[field.name]: 1765 if obj_list.has_key(wid): 1766 tmp = obj_list[wid] 1767 elif issubclass(field.rel.to, SpeciesComponent): 1768 tmp = field.rel.to.objects.get(species__wid=species_wid, wid=wid) 1769 else: 1770 tmp = field.rel.to.objects.get(wid=wid) 1771 getattr(obj, field.name).add(tmp) 1772 else: 1773 for sub_obj_data in obj_data[field.name]: 1774 getattr(obj, field.name).add(save_object_data(species_wid, field.rel.to(), sub_obj_data, obj_list, user, save=save, save_m2m=save_m2m)) 1775 1776 #save 1777 if save: 1778 obj.full_clean() 1779 obj.save() 1780 1781 #return obj 1782 return obj

1783

1784 -def format_sequence_as_html(sequence, lineLen=50):

1785 htmlL = '' 1786 htmlC = '' 1787 htmlR = '' 1788 1789 for i in range(int(ceil(float(len(sequence)) / float(lineLen)))): 1790 htmlL += '%s<br/>' % (i * lineLen + 1, ) 1791 htmlC += '%s<br/>' % sequence[i*lineLen:(i + 1) * lineLen] 1792 htmlR += '%s<br/>' % (min(len(sequence), (i + 1) * lineLen), ) 1793 1794 return '<div class="sequence"><div>%s</div><div>%s</div><div>%s</div></div>' % (htmlL, htmlC, htmlR)

1795

1796 -def readFasta(species_wid, filename, user):

1797 error_messages = [] 1798 1799 f = open(filename, 'r') 1800 data = f.readlines() 1801 f.close() 1802 1803 wid = None 1804 sequences = {} 1805 for i in range(len(data)): 1806 if data[i][0] == '>': 1807 wid = data[i].strip()[1:] 1808 sequences[wid] = '' 1809 else: 1810 if wid is None: 1811 return (False, 'File does not match FASTA format.') 1812 sequences[wid] += data[i].strip() 1813 1814 #retrieve chromosomes 1815 for wid, sequence in sequences.iteritems(): 1816 try: 1817 chr = Chromosome.objects.get(species__wid=species_wid, wid=wid) 1818 except ObjectDoesNotExist as error: 1819 error_messages.append(error.message) 1820 continue 1821 if len(error_messages) > 0: 1822 return (False, format_list_html(error_messages)) 1823 1824 #validate 1825 for wid, sequence in sequences.iteritems(): 1826 chr = Chromosome.objects.get(species__wid=species_wid, wid=wid) 1827 chr.sequence = sequence 1828 try: 1829 chr.full_clean() 1830 except ValidationError as error: 1831 error_messages.append(format_error_as_html_list(error)) 1832 continue 1833 if len(error_messages) > 0: 1834 return (False, format_list_html(error_messages)) 1835 1836 1837 #save 1838 for wid, sequence in sequences.iteritems(): 1839 chr = Chromosome.objects.get(species__wid=species_wid, wid=wid) 1840 chr.sequence = sequence 1841 chr.full_clean() 1842 try: 1843 chr.save() 1844 except ValueError as error: 1845 error_messages.append(error.message) 1846 continue 1847 if len(error_messages) > 0: 1848 return (False, format_list_html(error_messages)) 1849 1850 return (True, 'Sequences successfully saved!')

1851

1852 -def write_bibtex(species, qs):

1853 refs = {} 1854 for obj in qs: 1855 if not isinstance(obj, SpeciesComponent): 1856 continue 1857 1858 if isinstance(obj, Reference): 1859 refs[obj.wid] = obj 1860 for sub_obj in obj.references.all(): 1861 refs[sub_obj.wid] = sub_obj 1862 for field in obj.__class__._meta.fields + obj.__class__._meta.many_to_many: 1863 if isinstance(field, ForeignKey) and issubclass(field.rel.to, EvidencedEntryData) and getattr(obj, field.name) is not None: 1864 for evidence in getattr(obj, field.name).evidence.all(): 1865 for ref in evidence.references.all(): 1866 refs[ref.wid] = ref 1867 if isinstance(field, ManyToManyField) and issubclass(field.rel.to, EvidencedEntryData): 1868 for sub_obj in getattr(obj, field.name).all(): 1869 for evidence in sub_obj.evidence.all(): 1870 for ref in evidence.references.all(): 1871 refs[ref.wid] = ref 1872 1873 bibs = [] 1874 wids = refs.keys() 1875 wids.sort() 1876 for wid in wids: 1877 bibs.append(refs[wid].get_as_bibtex()) 1878 1879 now = datetime.datetime.now(tzlocal()) 1880 return \ 1881 '@preamble {"\n' \ 1882 + ('\t%s WholeCellKB Bibliography\n' % (species.name, )) \ 1883 + ('\tGenerated by WholeCellKB on %s at %s\n' % ( 1884 now.isoformat(), 1885 settings.ROOT_URL + reverse('public.views.exportData', kwargs={'species_wid': species.wid}), 1886 )) \ 1887 + ('\t%s %s %s\n' % (html_to_ascii('©'), now.year, 'Covert Lab, Department of Bioengineering, Stanford University', )) \ 1888 + '"}\n\n' \ 1889 + '\n\n'.join(bibs)

1890

1891 -def get_invalid_objects(species_id, validate_fields=False, validate_objects=True, full_clean=False, validate_unique=False):

1892 #check that species WIDs unique 1893 wids = [] 1894 for species in Species.objects.values('wid').all(): 1895 wids.append(species['wid']) 1896 if len(set(wids)) < len(wids): 1897 for wid in set(wids): 1898 del wids[wids.index(wid)] 1899 return [{ 1900 'type': 'model', 1901 'name': model_name, 1902 'url': None, 1903 'message': 'The following species WIDs are not unique:<ul><li>%s</li></ul>' % '</li></li>'.join(wids), 1904 }] 1905 1906 #retrieve objects 1907 all_obj_wids = {} 1908 all_obj_data = {} 1909 all_obj_data_by_model = {} 1910 1911 species = Species.objects.select_related().get(id=species_id) 1912 species_wid = species.wid 1913 all_obj_wids[species.wid] = 'Species' 1914 all_obj_data[species.wid] = species 1915 all_obj_data_by_model['Species'] = [species] 1916 1917 wids = [] 1918 for model_name, model in getModels(SpeciesComponent).iteritems(): 1919 all_obj_data_by_model[model_name] = [] 1920 for obj in model.objects.select_related().filter(species__id=species_id): 1921 wids.append(obj.wid) 1922 all_obj_wids[obj.wid] = model_name 1923 all_obj_data[obj.wid] = obj 1924 all_obj_data_by_model[model_name].append(obj) 1925 1926 #check that oobject WIDs unique 1927 if len(set(wids)) < len(wids): 1928 for wid in set(wids): 1929 del wids[wids.index(wid)] 1930 return [{ 1931 'type': 'model', 1932 'name': model_name, 1933 'url': None, 1934 'message': 'The following WIDs are not unique:<ul><li>%s</li></ul>' % '</li></li>'.join(wids), 1935 }] 1936 1937 #validate 1938 errors = [] 1939 for model_name, model in getModels().iteritems(): 1940 for obj in all_obj_data_by_model[model_name]: 1941 junk, obj_data = get_edit_form_fields(species_wid, model, obj=obj) 1942 obj_data['species'] = species_wid 19

Source Code for Module WholeCellKB.public.helpers