from __future__ import absolute_import, division, print_function from operator import itemgetter import iotbx.phil import iotbx.pdb import iotbx.mrcfile from cctbx import crystal from cctbx import maptbx from libtbx.utils import Sorry import sys, os from cctbx.array_family import flex from scitbx.math import matrix from copy import deepcopy from libtbx.utils import null_out import libtbx.callbacks # import dependency from libtbx import group_args from six.moves import range from six.moves import zip from cctbx.development.create_models_or_maps import get_map_from_map_coeffs master_phil = iotbx.phil.parse(""" input_files { seq_file = None .type = path .short_caption = Sequence file .help = Sequence file (unique chains only, \ 1-letter code, chains separated by \ blank line or greater-than sign.) \ Can have chains that are DNA/RNA/protein and\ all can be present in one file. \ If not supplied, must supply molecular mass or \ solvent content. map_file = None .type = path .help = File with CCP4-style map .short_caption = Map file half_map_file = None .type = path .multiple = True .short_caption = Half map .help = Half map (two should be supplied) for FSC calculation. Must \ have grid identical to map_file external_map_file = None .type = path .short_caption = External map .style = file_type:ccp4_map bold input_file .help = External map to be used to scale map_file (power vs resolution\ will be matched). Not used in segment_and_split_map ncs_file = None .type = path .help = File with symmetry information (typically point-group NCS with \ the center specified). Typically in PDB format. \ Can also be a .ncs_spec file from phenix. \ Created automatically if symmetry is specified. .short_caption = symmetry file pdb_file = None .type = path .help = Optional PDB file matching map_file to be offset pdb_to_restore = None .type = path .help = Optional PDB file to restore to position matching original \ map_file. Used in combination with info_file = xxx.pkl \ and restored_pdb = yyyy.pdb .short_caption = PDB to restore info_file = None .type = path .help = Optional pickle file with information from a previous run.\ Can be used with pdb_to_restore to restore a PDB file to \ to position matching original \ map_file. .short_caption = Info file target_ncs_au_file = None .type = path .help = Optional PDB file to partially define the ncs asymmetric \ unit of the map. The coordinates in this file will be used \ to mark part of the ncs au and all points nearby that are \ not part of another ncs au will be added. input_weight_map_pickle_file = None .type = path .short_caption = Input weight map pickle file .help = Weight map pickle file } output_files { magnification_map_file = magnification_map.ccp4 .type = path .help = Input map file with magnification applied. Only written if\ magnification is applied. .short_caption = Magnification map file magnification_ncs_file = magnification_ncs.ncs_spec .type = path .help = Input NCS with magnification applied. Only written if\ magnification is applied. .short_caption = Magnification NCS file shifted_map_file = shifted_map.ccp4 .type = path .help = Input map file shifted to new origin. .short_caption = Shifted map file shifted_sharpened_map_file = shifted_sharpened_map.ccp4 .type = path .help = Input map file shifted to new origin and sharpened. .short_caption = Shifted sharpened map file sharpened_map_file = sharpened_map.ccp4 .type = str .short_caption = Sharpened map file .help = Output sharpened map file, superimposed on the original map. .input_size = 400 shifted_pdb_file = shifted_pdb.pdb .type = path .help = Input pdb file shifted to new origin. .short_caption = Shifted pdb file shifted_ncs_file = shifted_ncs.ncs_spec .type = path .help = NCS information shifted to new origin. .short_caption = Output NCS info file shifted_used_ncs_file = shifted_used_ncs.ncs_spec .type = path .help = NCS information (just the part that is used) shifted \ to new origin. .short_caption = Output used NCS info file output_directory = segmented_maps .type = path .help = Directory where output files are to be written \ applied. .short_caption = Output directory box_map_file = box_map_au.ccp4 .type = path .help = Output map file with one NCS asymmetric unit, cut out box .short_caption = Box NCS map file box_mask_file = box_mask_au.ccp4 .type = path .help = Output mask file with one NCS asymmetric unit, cut out box .short_caption = Box NCS mask file box_buffer = 5 .type = int .help = Buffer (grid units) around NCS asymmetric unit in box_mask and map .short_caption = Box buffer size au_output_file_stem = shifted_au .type = str .help = File stem for output map files with one NCS asymmetric unit .short_caption = Output au file stem write_intermediate_maps = False .type = bool .help = Write out intermediate maps and masks for visualization .short_caption = Write intermediate maps write_output_maps = True .type = bool .help = Write out maps .short_caption = Write maps remainder_map_file = remainder_map.ccp4 .type = path .help = output map file with remainder after initial regions identified .short_caption = Output remainder map file output_info_file = segment_and_split_map_info.pkl .type = path .help = Output pickle file with information about map and masks .short_caption = Output pickle file restored_pdb = None .type = path .help = Output name of PDB restored to position matching original \ map_file. Used in combination with info_file = xxx.pkl \ and pdb_to_restore = xxxx.pdb .short_caption = Restored PDB file output_weight_map_pickle_file = weight_map_pickle_file.pkl .type = path .short_caption = Output weight map pickle file .help = Output weight map pickle file } crystal_info { chain_type = *None PROTEIN RNA DNA .type = choice .short_caption = Chain type .help = Chain type. Determined automatically from sequence file if \ not given. Mixed chain types are fine (leave blank if so). sequence = None .type = str .short_caption = Sequence .help = Sequence as string is_crystal = None .type = bool .short_caption = Is a crystal .help = Defines whether this is a crystal (or cryo-EM).\ Default is True if use_sg_symmetry = True and False otherwise. use_sg_symmetry = False .type = bool .short_caption = Use space-group symmetry .help = If you set use_sg_symmetry = True then the symmetry of the space\ group will be used. For example in P1 a point at one end of \ the \ unit cell is next to a point on the other end. Normally for \ cryo-EM data this should be set to False and for crystal data \ it should be set to True. This will normally also set the \ value of is_crystal (same value as use_sg_symmetry) and \ restrict_map_size (False if use_sg_symmetry = True). resolution = None .type = float .short_caption = resolution .help = Nominal resolution of the map. This is used later to decide on\ resolution cutoffs for Fourier inversion of the map. Note: \ the resolution is not cut at this value, it is cut at \ resolution*d_min_ratio if at all. space_group = None .type = space_group .help = Space group (used for boxed maps) .style = hidden unit_cell = None .type = unit_cell .help = Unit Cell (used for boxed maps) .style = hidden original_unit_cell = None .type = unit_cell .help = Original unit cell (of input map). Used internally .style = hidden original_unit_cell_grid = None .type = ints .help = Original unit cell grid (of input map). Used internally .style = hidden molecular_mass = None .type = float .help = Molecular mass of molecule in Da. Used as alternative method \ of specifying solvent content. .short_caption = Molecular mass in Da solvent_content = None .type = float .help = Solvent fraction of the cell. Used for ID of \ solvent content in boxed maps. .short_caption = Solvent content solvent_content_iterations = 3 .type = int .help = Iterations of solvent fraction estimation. Used for ID of \ solvent content in boxed maps. .short_caption = Solvent fraction iterations .style = hidden wang_radius = None .type = float .help = Wang radius for solvent identification. \ Default is 1.5* resolution .short_caption = Wang radius buffer_radius = None .type = float .help = Buffer radius for mask smoothing. \ Default is resolution .short_caption = Buffer radius pseudo_likelihood = None .type = bool .help = Use pseudo-likelihood method for half-map sharpening. \ (In development) .short_caption = Pseudo-likelihood .style = hidden } reconstruction_symmetry { symmetry = None .type = str .short_caption = Symmetry type .help = Symmetry used in reconstruction. For example D7, C3, C2\ I (icosahedral), T (tetrahedral), or ANY (try everything and \ use the highest symmetry found). Not needed if ncs_file is supplied. \ include_helical_symmetry = True .type = bool .short_caption = Include helical symmetry .help = You can include or exclude searches for helical symmetry must_be_consistent_with_space_group_number = None .type = int .short_caption = Space group to match .help = Searches for symmetry must be compatible with this space group\ number. symmetry_center = None .type = floats .short_caption = symmetry center .help = Center (in A) for symmetry operators (if symmetry is found \ automatically). \ If set to None, first guess is the center of the cell and then \ if that fails, found automatically as the center of the \ density in the map. optimize_center = False .type = bool .short_caption = Optimize symmetry center .help = Optimize position of symmetry center. Also checks for center \ at (0, 0, 0) vs center of map helical_rot_deg = None .type = float .short_caption = helical rotation .help = helical rotation about z in degrees helical_trans_z_angstrom = None .type = float .short_caption = helical translation .help = helical translation along z in Angstrom units max_helical_optimizations = 2 .type = int .short_caption = Max helical optimizations .help = Number of optimizations of helical parameters\ when finding symmetry max_helical_ops_to_check = 5 .type = int .short_caption = Max helical ops to check .help = Number of helical operations in each direction to check \ when finding symmetry max_helical_rotations_to_check = None .type = int .short_caption = Max helical rotations .help = Number of helical rotations to check \ when finding symmetry two_fold_along_x = None .type = bool .short_caption = D two-fold along x .help = Specifies if D or I two-fold is along x (True) or y (False). \ If None, both are tried. smallest_object = None .type = float .short_caption = Smallest object to consider .help = Dimension of smallest object to consider\ when finding symmetry. Default is 5 * resolution score_basis = ncs_score cc *None .type = choice .short_caption = Symmetry score basis .help = Symmetry score basis. Normally ncs_score (sqrt(n)* cc) is \ used except for identification of helical symmetry scale_weight_fractional_translation = 1.05 .type = float .short_caption = Scale on fractional translation .help = Give slight increase in weighting in helical symmetry \ search to translations that are a fraction (1/2, 1/3) of \ the d-spacing of the peak of intensity in the fourier \ transform of the density. random_points = 100 .type = int .short_caption = Random points .help = Number of random points in map to examine in finding symmetry identify_ncs_id = True .type = bool .short_caption = Identify NCS ID .help = If symmetry is not point-group symmetry, try each possible \ operator when evaluating symmetry and choose the one that \ results in the most uniform density at symmetry-related points. min_ncs_cc = 0.75 .type = float .short_caption = Minimum symmetry CC to keep it .help = Minimum symmetry CC to keep operators when identifying \ automatically n_rescore = 5 .type = int .short_caption = symmetry operators to rescore .help = Number of symmetry operators to rescore op_max = 14 .type = int .short_caption = Max operators to try .help = If symmetry is ANY, try up to op_max-fold symmetries tol_r = 0.02 .type = float .help = tolerance in rotations for point group or helical symmetry .short_caption = Rotation tolerance abs_tol_t = 2 .type = float .help = tolerance in translations (A) for point group or helical symmetry .short_caption = Translation tolerance absolute max_helical_operators = None .type = int .help = Maximum helical operators (if extending existing helical\ operators) .short_caption = Maximum helical operators rel_tol_t = .05 .type = float .help = tolerance in translations (fractional) for point group or \ helical symmetry .short_caption = Translation tolerance fractional require_helical_or_point_group_symmetry = False .type = bool .help = normally helical or point-group symmetry (or none) is expected. \ However in some cases (helical + rotational symmetry for \ example) this is not needed and is not the case. .short_caption = Require helical or point-group or no symmetry } map_modification { magnification = None .type = float .short_caption = Magnification .help = Magnification to apply to input map. Input map grid will be \ scaled by magnification factor before anything else is done. b_iso = None .type = float .short_caption = Target b_iso .help = Target B-value for map (sharpening will be applied to yield \ this value of b_iso). If sharpening method is not supplied, \ default is to use b_iso_to_d_cut sharpening. b_sharpen = None .type = float .short_caption = Sharpening .help = Sharpen with this b-value. Contrast with b_iso that yield a \ targeted value of b_iso. B_sharpen greater than zero is sharpening.\ Less than zero is blurring. b_blur_hires = 200 .type = float .short_caption = high_resolution blurring .help = Blur high_resolution data (higher than d_cut) with \ this b-value. Contrast with b_sharpen applied to data up to\ d_cut. \ Note on defaults: If None and b_sharpen is positive (sharpening) \ then high-resolution data is left as is (not sharpened). \ If None and b_sharpen is negative (blurring) high-resolution data\ is also blurred. resolution_dependent_b = None .type = floats .short_caption = resolution_dependent b .help = If set, apply resolution_dependent_b (b0 b1 b2). \ Log10(amplitudes) will start at 1, change to b0 at half \ of resolution specified, changing linearly, \ change to b1/2 at resolution specified, \ and change to b1/2+b2 at d_min_ratio*resolution normalize_amplitudes_in_resdep = False .type = bool .short_caption = Normalize amplitudes in resdep .help = Normalize amplitudes in resolution-dependent sharpening d_min_ratio = 0.833 .type = float .short_caption = Sharpen d_min ratio .help = Sharpening will be applied using d_min equal to \ d_min_ratio times resolution. Default is 0.833 scale_max = 100000 .type = float .short_caption = Scale_max .help = Scale amplitudes from inverse FFT to yield maximum of this value input_d_cut = None .type = float .short_caption = d_cut .help = High-resolution limit for sharpening rmsd = None .type = float .short_caption = RMSD of model .help = RMSD of model to true model (if supplied). Used to \ estimate expected fall-of with resolution of correct part \ of model-based map. If None, assumed to be resolution \ times rmsd_resolution_factor. rmsd_resolution_factor = 0.25 .type = float .short_caption = rmsd resolution factor .help = default RMSD is resolution times resolution factor fraction_complete = None .type = float .short_caption = Completeness model .help = Completness of model (if supplied). Used to \ estimate correct part \ of model-based map. If None, estimated from max(FSC). regions_to_keep = None .type = int .short_caption = Regions to keep .help = You can specify a limit to the number of regions to keep\ when generating the asymmetric unit of density. auto_sharpen = True .type = bool .short_caption = Automatically determine sharpening .help = Automatically determine sharpening using kurtosis maximization\ or adjusted surface area auto_sharpen_methods = no_sharpening b_iso *b_iso_to_d_cut \ resolution_dependent model_sharpening \ half_map_sharpening target_b_iso_to_d_cut None .type = choice(multi = True) .short_caption = Sharpening methods .help = Methods to use in sharpening. b_iso searches for b_iso to \ maximize sharpening target (kurtosis or adjusted_sa). \ b_iso_to_d_cut applies b_iso only up to resolution specified, with \ fall-over of k_sharpen. Resolution dependent adjusts 3 parameters \ to sharpen variably over resolution range. Default is \ b_iso_to_d_cut . target_b_iso_to_d_cut uses target_b_iso_ratio \ to set b_iso. box_in_auto_sharpen = False .type = bool .short_caption = Use box for auto_sharpening .help = Use a representative box of density for initial \ auto-sharpening instead of the entire map. density_select_in_auto_sharpen = True .type = bool .short_caption = density_select to choose box .help = Choose representative box of density for initial \ auto-sharpening with density_select method \ (choose region where there is high density). \ Normally use this as well as density_select = True which \ carries out density_select at start of segmentation. density_select_threshold_in_auto_sharpen = None .type = float .short_caption = density_select threshold to choose box .help = Threshold for density select choice of box. Default is 0.05. \ If your map has low overall contrast you might need to make this\ bigger such as 0.2. allow_box_if_b_iso_set = False .type = bool .short_caption = Allow box if b_iso set .help = Allow box_in_auto_sharpen (if set to True) even if \ b_iso is set. Default is to set box_n_auto_sharpen = False \ if b_iso is set. soft_mask = True .type = bool .help = Use soft mask (smooth change from inside to outside with radius\ based on resolution of map). Required if you use half-map \ sharpening without a model, otherwise optional. .short_caption = Soft mask use_weak_density = False .type = bool .short_caption = Use box with poor density .help = When choosing box of representative density, use poor \ density (to get optimized map for weaker density) discard_if_worse = None .type = bool .short_caption = Discard sharpening if worse .help = Discard sharpening if worse local_sharpening = None .type = bool .short_caption = Local sharpening .help = Sharpen locally using overlapping regions. \ NOTE: Best to turn off local_aniso_in_local_sharpening \ if symmetry is present.\ If local_aniso_in_local_sharpening is True and symmetry is \ present this can distort the map for some symmetry copies \ because an anisotropy correction is applied\ based on local density in one copy and is transferred without \ rotation to other copies. local_aniso_in_local_sharpening = None .type = bool .short_caption = Local anisotropy .help = Use local anisotropy in local sharpening. \ Default is True unless symmetry is present. overall_before_local = True .type = bool .short_caption = Overall before local .help = Apply overall scaling before local scaling select_sharpened_map = None .type = int .short_caption = Sharpened map to use .help = Select a single sharpened map to use read_sharpened_maps = None .type = bool .short_caption = Read sharpened maps .help = Read in previously-calculated sharpened maps write_sharpened_maps = None .type = bool .short_caption = Write sharpened maps .help = Write out local sharpened maps smoothing_radius = None .type = float .short_caption = Smoothing radius .help = Sharpen locally using smoothing_radius. Default is 2/3 of \ mean distance between centers for sharpening box_center = None .type = floats .short_caption = Center of box .help = You can specify the center of the box (A units) box_size = 40 40 40 .type = ints .short_caption = Size of box .help = You can specify the size of the boxes to use (grid units) target_n_overlap = 10 .type = int .short_caption = Target overlap of boxes .help = You can specify the targeted overlap of boxes in local \ sharpening restrict_map_size = None .type = bool .short_caption = Restrict box map size .help = Restrict box map to be inside full map (required for cryo-EM data).\ Default is True if use_sg_symmetry = False. restrict_z_turns_for_helical_symmetry = 1 .type = float .short_caption = Restrict Z turns for helical symmetry .help = Restrict Z turns for helical symmetry. Number of \ turns of helix going each direction in Z is specified. restrict_z_distance_for_helical_symmetry = None .type = float .short_caption = Restrict Z distance for helical symmetry .help = Restrict Z distance (+/- this distance from center) \ for helical symmetry. remove_aniso = True .type = bool .short_caption = Remove aniso .help = You can remove anisotropy (overall and locally) during sharpening cc_cut = 0.2 .type = float .short_caption = Min reliable CC in half-maps .help = Estimate of minimum highly reliable CC in half-map FSC. Used\ to decide at what CC value to smooth the remaining CC values. max_cc_for_rescale = 0.2 .type = float .short_caption = Max CC for rescale .help = Min reliable CC in half-maps. \ Used along with cc_cut and scale_using_last to correct for \ small errors in FSC estimation at high resolution. If the \ value of FSC near the high-resolution limit is above \ max_cc_for_rescale, assume these values are correct and do not \ correct them. To keep all original values use\ max_cc_for_rescale = 1 scale_using_last = 3 .type = int .short_caption = Last N bins in FSC assumed to be about zero .help = If set, assume that the last scale_using_last bins in the FSC \ for half-map or model sharpening are about zero (corrects for \ errors in the half-map process). max_box_fraction = 0.5 .type = float .short_caption = Max size of box for auto_sharpening .help = If box is greater than this fraction of entire map, use \ entire map. density_select_max_box_fraction = 0.95 .type = float .short_caption = Max size of box for density_select .help = If box is greater than this fraction of entire map, use \ entire map for density_select. Default is 0.95 mask_atoms = True .type = bool .short_caption = Mask atoms .help = Mask atoms when using model sharpening mask_atoms_atom_radius = 3 .type = float .short_caption = Mask radius .help = Mask for mask_atoms will have mask_atoms_atom_radius value_outside_atoms = None .type = str .short_caption = Value outside atoms .help = Value of map outside atoms (set to 'mean' to have mean \ value inside and outside mask be equal) k_sharpen = 10 .type = float .short_caption = sharpening transition .help = Steepness of transition between sharpening (up to resolution \ ) and not sharpening (d < resolution). Note: for blurring, \ all data are blurred (regardless of resolution), while for \ sharpening, only data with d about resolution or lower are \ sharpened. This prevents making very high-resolution data too \ strong. Note 2: if k_sharpen is zero, then no \ transition is applied and all data is sharpened or blurred. \ Note 3: only used if b_iso is set. iterate = False .type = bool .short_caption = Iterate auto-sharpening .help = You can iterate auto-sharpening. This is useful in cases where \ you do not specify the solvent content and it is not \ accurately estimated until sharpening is optimized. optimize_b_blur_hires = False .type = bool .short_caption = Optimize value of b_blur_hires .help = Optimize value of b_blur_hires. \ Only applies for auto_sharpen_methods b_iso_to_d_cut and \ b_iso. This is normally carried out and helps prevent \ over-blurring at high resolution if the same map is \ sharpened more than once. optimize_d_cut = None .type = bool .short_caption = Optimize value of d_cut .help = Optimize value of d_cut. \ Only applies for auto_sharpen_methods b_iso_to_d_cut and \ b_iso. Not normally carried out. adjust_region_weight = True .type = bool .short_caption = Adjust region weight .help = Adjust region_weight to make overall change in surface area \ equal to overall change in normalized regions over the range \ of search_b_min to search_b_max using b_iso_to_d_cut. region_weight_method = initial_ratio *delta_ratio b_iso .type = choice .short_caption = Region weight method .help = Method for choosing region_weights. Initial_ratio uses \ ratio of surface area to regions at low B value. Delta \ ratio uses change in this ratio from low to high B. B_iso \ uses resolution-dependent b_iso (not weights) with the \ formula b_iso = 5.9*d_min**2 region_weight_factor = 1.0 .type = float .short_caption = Region weight factor .help = Multiplies region_weight after calculation with \ region_weight_method above region_weight_buffer = 0.1 .type = float .short_caption = Region weight factor buffer .help = Region_weight adjusted to be region_weight_buffer \ away from minimum or maximum values region_weight_default = 30. .type = float .short_caption = Region weight default .help = Region_weight adjusted to be region_weight_default\ if no information available target_b_iso_ratio = 5.9 .type = float .short_caption = Target b_iso ratio .help = Target b_iso ratio : b_iso is estimated as \ target_b_iso_ratio * resolution**2 signal_min = 3.0 .type = float .short_caption = Minimum signal .help = Minimum signal in estimation of optimal b_iso. If\ not achieved, use any other method chosen. target_b_iso_model_scale = 0. .type = float .short_caption = scale on target b_iso ratio for model .help = For model sharpening, the target_biso is scaled \ (normally zero). search_b_min = -100 .type = float .short_caption = Low bound for b_iso search .help = Low bound for b_iso search. search_b_max = 300 .type = float .short_caption = High bound for b_iso search .help = High bound for b_iso search. search_b_n = 21 .type = int .short_caption = Number of b_iso values to search .help = Number of b_iso values to search. residual_target = 'adjusted_sa' .type = str .short_caption = Residual target .help = Target for maximization steps in sharpening. \ Can be kurtosis or adjusted_sa (adjusted surface area) sharpening_target = 'adjusted_sa' .type = str .short_caption = Overall sharpening target .help = Overall target for sharpening. Can be kurtosis or adjusted_sa \ (adjusted surface area) or adjusted_path_length. \ Used to decide which sharpening approach \ is used. Note that during optimization, residual_target is used \ (they can be the same.) region_weight = 40 .type = float .short_caption = Region weighting .help = Region weighting in adjusted surface area calculation.\ Score is surface area minus region_weight times number of regions.\ Default is 40. A smaller value will give more sharpening. sa_percent = 30. .type = float .short_caption = Percent of target regions in adjusted_sa .help = Percent of target regions used in calulation of adjusted \ surface area. Default is 30. fraction_occupied = 0.20 .type = float .short_caption = Fraction of molecular volume inside contours .help = Fraction of molecular volume targeted to be inside contours. \ Used to set contour level. Default is 0.20 n_bins = 20 .type = int .short_caption = Resolution bins .help = Number of resolution bins for sharpening. Default is 20. max_regions_to_test = 30 .type = int .short_caption = Max regions to test .help = Number of regions to test for surface area in adjusted_sa \ scoring of sharpening eps = None .type = float .short_caption = Shift used in calculation of derivatives for \ sharpening maximization. Default is 0.01 for kurtosis and 0.5 for \ adjusted_sa. k_sol = 0.35 .type = float .help = k_sol value for model map calculation. IGNORED (Not applied) .short_caption = k_sol IGNORED .style = hidden b_sol = 50 .type = float .help = b_sol value for model map calculation. IGNORED (Not applied) .short_caption = b_sol IGNORED .style = hidden } segmentation { select_au_box = None .type = bool .help = Select box containing at least one representative region of \ the map. Also select just symmetry operators relevant to that box. \ Default is true if number of operators is at least \ n_ops_to_use_au_box .short_caption = select au box n_ops_to_use_au_box = 25 .type = int .help = If number of operators is this big or more and \ select_au_box is None, set it to True. .short_caption = N ops to use au_box n_au_box = 5 .type = int .help = Number of symmetry copies to try and get inside au_box .short_caption = N au box lower_bounds = None .type = ints .help = You can select a part of your map for analysis with \ lower_bounds and upper_bounds. .short_caption = Lower bounds upper_bounds = None .type = ints .help = You can select a part of your map for analysis with \ lower_bounds and upper_bounds. .short_caption = Upper bounds density_select = True .type = bool .help = Run map_box with density_select = True to cut out the region \ in the input map that contains density. Useful if the input map \ is much larger than the structure. Done before segmentation is\ carried out. .short_caption = Trim map to density density_select_threshold = 0.05 .type = float .help = Choose region where density is this fraction of maximum or greater .short_caption = threshold for density_select get_half_height_width = None .type = bool .help = Use 4 times half-width at half-height as estimate of max size .short_caption = Half-height width estimation box_ncs_au = True .type = bool .help = Box the map containing just the au of the map .short_caption = Box NCS au cell_cutoff_for_solvent_from_mask = 150 .type = float .help = For cells with average edge over this cutoff, use the\ low resolution mask (backup) method for solvent estimation .short_caption = Cell cutoff for solvent_from_mask mask_padding_fraction = 0.025 .type = float .help = Adjust threshold of standard deviation map in low resolution \ mask identification of solvent content to give this much more \ inside mask than would be obtained with the value of\ fraction_of_max_mask_threshold. .short_caption = Mask padding fraction fraction_of_max_mask_threshold = .05 .type = float .help = threshold of standard deviation map in low resolution mask \ identification of solvent content. .short_caption = Fraction of max mask_threshold mask_threshold = None .type = float .help = threshold in identification of overall mask. If None, guess \ volume of molecule from sequence and symmetry copies. .short_caption = Density select threshold grid_spacing_for_au = 3 .type = int .help = Grid spacing for asymmetric unit when constructing asymmetric unit. .short_caption = Grid spacing for constructing asymmetric unit radius = None .type = float .help = Radius for constructing asymmetric unit. .short_caption = Radius for constructing asymmetric unit value_outside_mask = 0.0 .type = float .help = Value to assign to density outside masks .short_caption = Value outside mask density_threshold = None .type = float .short_caption = Density threshold .help = Threshold density for identifying regions of density. \ Applied after normalizing the density in the region of \ the molecule to an rms of 1 and mean of zero. starting_density_threshold = None .type = float .short_caption = Starting density threshold .help = Optional guess of threshold density iteration_fraction = 0.2 .type = float .short_caption = Iteration fraction .help = On iteration of finding regions, assume target volume is \ this fraction of the value on previous iteration max_overlap_fraction = 0.05 .type = float .short_caption = Max overlap .help = Maximum fractional overlap allowed to density in another \ asymmetric unit. Definition of a bad region. remove_bad_regions_percent = 1 .type = float .short_caption = Remove worst overlapping regions .help = Remove the worst regions that are part of more than one NCS \ asymmetric unit, up to remove_bad_regions_percent of the total require_complete = True .type = bool .short_caption = Require all symmetry copies to be represented for a region .help = Require all symmetry copies to be represented for a region split_if_possible = True .type = bool .short_caption = Split regions if mixed .help = Split regions that are split in some symmetry copies.\ If None, split if most copies are split. write_all_regions = False .type = bool .short_caption = Write all regions .help = Write all regions to ccp4 map files. max_per_au = None .type = int .short_caption = Max regions in au .help = Maximum number of regions to be kept in the NCS asymmetric unit max_per_au_ratio = 5. .type = int .short_caption = Max ratio of regions to expected .help = Maximum ratio of number of regions to be kept in the \ NCS asymmetric unit to those expected min_ratio_of_ncs_copy_to_first = 0.5 .type = float .short_caption = Minimum ratio of ncs copy to first .help = Minimum ratio of the last ncs_copy region size to maximum min_ratio = 0.1 .type = float .short_caption = Minimum ratio to keep .help = Minimum ratio of region size to maximum to keep it max_ratio_to_target = 3 .type = float .help = Maximum ratio of grid points in top region to target .short_caption = Max ratio to target min_ratio_to_target = 0.3 .type = float .help = Minimum ratio of grid points in top region to target .short_caption = Min ratio to target min_volume = 10 .type = int .help = Minimum region size to consider (in grid points) .short_caption = Minimum region size residues_per_region = 50 .type = float .help = Target number of residues per region .short_caption = Residues per region seeds_to_try = 10 .type = int .help = Number of regions to try as centers .short_caption = Seeds to try iterate_with_remainder = True .type = bool .short_caption = Iterate .help = Iterate looking for regions based on remainder from first analysis weight_rad_gyr = 0.1 .type = float .short_caption = Weight on radius of gyration .help = Weight on radius of gyration of group of regions in NCS AU \ relative to weight on closeness to neighbors. Normalized to\ largest cell dimension with weight = weight_rad_gyr*300/cell_max expand_size = None .type = int .help = Grid points to expand size of regions when excluding for next \ round. If None, set to approx number of grid points to get \ expand_target below .short_caption = Expand size expand_target = 1.5 .type = float .help = Target expansion of regions (A) .short_caption = Expand target mask_additional_expand_size = 1 .type = int .help = Mask expansion in addition to expand_size for final map .short_caption = Mask additional expansion mask_expand_ratio = 1 .type = int .help = Mask expansion relative to resolution for save_box_map_ncs_au .short_caption = Mask expand ratio exclude_points_in_ncs_copies = True .type = bool .help = Exclude points that are in symmetry copies when creating NCS au. \ Does not apply if add_neighbors = True .short_caption = Exclude points in symmetry copies add_neighbors = True .type = bool .help = Add neighboring regions around the au. Turns off \ exclude_points_in_ncs_copies also. .short_caption = Add neighbors add_neighbors_dist = 1. .type = float .help = Max increase in radius of gyration by adding region to keep it. .short_caption = Add neighbors dist } control { verbose = False .type = bool .help = '''Verbose output''' .short_caption = Verbose output shift_only = None .type = bool .short_caption = Shift only .help = Shift map and half_maps and stop sharpen_only = None .type = bool .short_caption = Sharpen only .help = Sharpen map and stop check_ncs = None .type = bool .short_caption = Check NCS .help = Check the NCS symmetry by estimating NCS correlation and stop resolve_size = None .type = int .help = "Size of resolve to use. " .style = hidden quick = True .type = bool .help = Run quickly if possible .short_caption = Quick run memory_check = True .type = bool .help = Map-to-model checks to make sure you have enough memory on \ your machine to run. You can disable this by setting this \ keyword to False. The estimates are approximate so it is \ possible your job could run even if the check fails. Note \ the check does not take any other uses of the memory on \ your machine into account. .short_caption = Memory check save_box_map_ncs_au = False .type = bool .help = Controls whether the map_box ncs_au is saved. Internal use only .style = hidden write_files = True .type = bool .help = Controls whether files are written .short_caption = Write files multiprocessing = *multiprocessing sge lsf pbs condor pbspro slurm .type = choice .short_caption = multiprocessing type .help = Choices are multiprocessing (single machine) or queuing systems queue_run_command = None .type = str .short_caption = Queue run command .help = run command for queue jobs. For example qsub. nproc = 1 .type = int .short_caption = Number of processors .help = Number of processors to use .style = renderer:draw_nproc_widget bold } """, process_includes = True) master_params = master_phil class map_and_b_object: def __init__(self, map_data = None, starting_b_iso = None, final_b_iso = None): from libtbx import adopt_init_args adopt_init_args(self, locals()) class pdb_info_object: def __init__(self, file_name = None, n_residues = None, ): from libtbx import adopt_init_args adopt_init_args(self, locals()) import time self.init_asctime = time.asctime() def show_summary(self, out = sys.stdout): print("PDB file:%s" %(self.file_name), end = ' ', file = out) if self.n_residues: print(" Residues: %d" %(self.n_residues), file = out) else: print(file = out) class seq_info_object: def __init__(self, file_name = None, sequence = None, n_residues = None, ): from libtbx import adopt_init_args adopt_init_args(self, locals()) import time self.init_asctime = time.asctime() def show_summary(self, out = sys.stdout): if self.file_name: print("Sequence file:%s" %(self.file_name), end = ' ', file = out) if self.n_residues: print(" Residues: %d" %(self.n_residues), file = out) else: print(file = out) class ncs_info_object: def __init__(self, file_name = None, number_of_operators = None, is_helical_symmetry = None, original_number_of_operators = None, ): from libtbx import adopt_init_args adopt_init_args(self, locals()) import time self.init_asctime = time.asctime() if original_number_of_operators is None: self.original_number_of_operators = number_of_operators self._has_updated_operators = False def show_summary(self, out = sys.stdout): print("NCS file:%s Operators: %d" %(self.file_name, self.number_of_operators), file = out) if self.is_helical_symmetry: print("Helical symmetry is present", file = out) def has_updated_operators(self): return self._has_updated_operators def update_number_of_operators(self, number_of_operators = None): self.number_of_operators = number_of_operators self._has_updated_operators = True def update_is_helical_symmetry(self, is_helical_symmetry = None): self.is_helical_symmetry = is_helical_symmetry self._has_updated_operators = True class map_info_object: def __init__(self, file_name = None, origin = None, all = None, crystal_symmetry = None, is_map = None, map_id = None, b_sharpen = None, id = None, ): from libtbx import adopt_init_args adopt_init_args(self, locals()) import time self.init_asctime = time.asctime() def show_summary(self, out = sys.stdout): if self.is_map: print("Map file:%s" %(self.file_name), end = ' ', file = out) else: print("Mask file:%s" %(self.file_name), end = ' ', file = out) if self.id is not None: print("ID: %d" %(self.id), end = ' ', file = out) if self.b_sharpen is not None: print("B-sharpen: %7.2f" %(self.b_sharpen), end = ' ', file = out) if self.map_id is not None: print("Map ID: %s" %(self.map_id), file = out) else: print(file = out) if self.origin and self.all: print(" Origin: %d %d %d Extent: %d %d %d" %( tuple(self.origin)+tuple(self.all)), file = out) if self.crystal_symmetry: print(" Map unit cell: %.1f %.1f %.1f %.1f %.1f %.1f " %( self.crystal_symmetry.unit_cell().parameters()), file = out) def lower_upper_bounds(self): lower_bounds = self.origin upper_bounds = [] for a, b in zip(self.origin, self.all): upper_bounds.append(a+b-1) # 2019-11-05 upper bound is na-1 return list(self.origin), list(upper_bounds) class info_object: def __init__(self, acc = None, ncs_obj = None, min_b = None, max_b = None, b_sharpen = None, # b_sharpen applied to map ncs_group_list = None, origin_shift = None, crystal_symmetry = None, # after density_select original_crystal_symmetry = None, # before density_select full_crystal_symmetry = None, # from real_map object full_unit_cell_grid = None, # from real_map object edited_volume_list = None, region_range_dict = None, selected_regions = None, ncs_related_regions = None, self_and_ncs_related_regions = None, map_files_written = None, bad_region_list = None, region_centroid_dict = None, original_id_from_id = None, remainder_id_dict = None, # dict relating regions in a remainder object to params = None, # input params input_pdb_info = None, input_map_info = None, input_ncs_info = None, input_seq_info = None, shifted_pdb_info = None, shifted_map_info = None, shifted_ncs_info = None, shifted_used_ncs_info = None, n_residues = None, solvent_fraction = None, output_ncs_au_map_info = None, output_ncs_au_mask_info = None, output_ncs_au_pdb_info = None, output_box_map_info = None, output_box_mask_info = None, output_region_map_info_list = None, output_region_pdb_info_list = None, sharpening_info_obj = None, box_map_bounds_first = None, box_map_bounds_last = None, final_output_sharpened_map_file = None, box_map_ncs_au = None, box_map_ncs_au_crystal_symmetry = None, ): if not selected_regions: selected_regions = [] if not ncs_related_regions: ncs_related_regions = [] if not self_and_ncs_related_regions: self_and_ncs_related_regions = [] if not map_files_written: map_files_written = [] if not output_region_map_info_list: output_region_map_info_list = [] if not output_region_pdb_info_list: output_region_pdb_info_list = [] from libtbx import adopt_init_args adopt_init_args(self, locals()) self.object_type = "segmentation_info" import time self.init_asctime = time.asctime() def set_box_map_ncs_au_map_data(self, box_map_ncs_au_map_data = None, box_mask_ncs_au_map_data = None, box_map_ncs_au_half_map_data_list = None, box_map_ncs_au_crystal_symmetry = None): self.box_map_ncs_au_map_data = box_map_ncs_au_map_data.deep_copy() self.box_mask_ncs_au_map_data = box_mask_ncs_au_map_data.deep_copy() self.box_map_ncs_au_half_map_data_list = [] for hm in box_map_ncs_au_half_map_data_list: self.box_map_ncs_au_half_map_data_list.append(hm.deep_copy()) self.box_map_ncs_au_crystal_symmetry = box_map_ncs_au_crystal_symmetry if self.origin_shift and self.origin_shift != (0, 0, 0): self.box_map_ncs_au_map_data = self.shift_map_back( map_data = self.box_map_ncs_au_map_data, crystal_symmetry = self.box_map_ncs_au_crystal_symmetry, shift_cart = self.origin_shift) self.box_mask_ncs_au_map_data = self.shift_mask_back( mask_data = self.box_mask_ncs_au_map_data, crystal_symmetry = self.box_mask_ncs_au_crystal_symmetry, shift_cart = self.origin_shift) new_hm_list = [] for hm in self.box_map_ncs_au_half_map_data_list: hm = self.shift_map_back( map_data = hm, crystal_symmetry = self.box_map_ncs_au_crystal_symmetry, shift_cart = self.origin_shift) new_hm_list.append(hm) self.box_map_ncs_au_half_map_data_list = new_hm_list def shift_map_back(self, map_data = None, crystal_symmetry = None, shift_cart = None): from scitbx.matrix import col new_origin = self.origin_shift_grid_units(crystal_symmetry = crystal_symmetry, map_data = map_data, shift_cart = shift_cart, reverse = True) new_all = list(col(map_data.all())+col(new_origin)) shifted_map_data = map_data.deep_copy() shifted_map_data.resize(flex.grid(new_origin, new_all)) return shifted_map_data def origin_shift_grid_units(self, crystal_symmetry = None, map_data = None, shift_cart = None, reverse = False): # Get origin shift in grid units from shift_cart from scitbx.matrix import col cell = crystal_symmetry.unit_cell().parameters()[:3] origin_shift_grid = [] for s, c, a in zip(shift_cart, cell, map_data.all()): if s<0: delta = -0.5 else: delta = 0.5 origin_shift_grid.append( int(delta+ a*s/c)) if reverse: return list(-col(origin_shift_grid)) else: return origin_shift_grid def is_segmentation_info_object(self): return True def set_params(self, params): self.params = deepcopy(params) def set_input_seq_info(self, file_name = None, sequence = None, n_residues = None): self.input_seq_info = seq_info_object(file_name = file_name, sequence = sequence, n_residues = n_residues) def set_input_pdb_info(self, file_name = None, n_residues = None): self.input_pdb_info = pdb_info_object(file_name = file_name, n_residues = n_residues) def set_input_ncs_info(self, file_name = None, number_of_operators = None): self.input_ncs_info = ncs_info_object(file_name = file_name, number_of_operators = number_of_operators) def update_ncs_info(self, number_of_operators = None, is_helical_symmetry = None, shifted = False): if shifted: ncs_info = self.shifted_ncs_info else: ncs_info = self.input_ncs_info assert ncs_info if number_of_operators is not None: ncs_info.update_number_of_operators( number_of_operators = number_of_operators) if is_helical_symmetry is not None: ncs_info.update_is_helical_symmetry( is_helical_symmetry = is_helical_symmetry) def set_sharpening_info(self, sharpening_info_obj = None): self.sharpening_info_obj = sharpening_info_obj def set_input_map_info(self, file_name = None, crystal_symmetry = None, origin = None, all = None): self.input_map_info = map_info_object(file_name = file_name, crystal_symmetry = crystal_symmetry, origin = origin, all = all, is_map = True) def set_ncs_obj(self, ncs_obj = None): self.ncs_obj = ncs_obj def set_origin_shift(self, origin_shift = None): if not origin_shift: origin_shift = (0, 0, 0) self.origin_shift = tuple(origin_shift) def set_crystal_symmetry(self, crystal_symmetry): self.crystal_symmetry = deepcopy(crystal_symmetry) def set_original_crystal_symmetry(self, crystal_symmetry): self.original_crystal_symmetry = deepcopy(crystal_symmetry) def set_full_crystal_symmetry(self, crystal_symmetry): self.full_crystal_symmetry = deepcopy(crystal_symmetry) def set_full_unit_cell_grid(self, unit_cell_grid): self.full_unit_cell_grid = deepcopy(unit_cell_grid) def set_box_map_bounds_first_last(self, box_map_bounds_first, box_map_bounds_last): self.box_map_bounds_first = box_map_bounds_first self.box_map_bounds_last = [] for l in box_map_bounds_last: self.box_map_bounds_last.append(l+1) # it is one bigger... def set_accessor(self, acc): self.acc = acc def set_shifted_map_info(self, file_name = None, crystal_symmetry = None, origin = None, all = None, b_sharpen = None): self.shifted_map_info = map_info_object(file_name = file_name, crystal_symmetry = crystal_symmetry, origin = origin, all = all, b_sharpen = b_sharpen, is_map = True) def set_shifted_pdb_info(self, file_name = None, n_residues = None): self.shifted_pdb_info = pdb_info_object(file_name = file_name, n_residues = n_residues) def set_shifted_ncs_info(self, file_name = None, number_of_operators = None, is_helical_symmetry = None): self.shifted_ncs_info = ncs_info_object(file_name = file_name, number_of_operators = number_of_operators, is_helical_symmetry = is_helical_symmetry) def set_shifted_used_ncs_info(self, file_name = None, number_of_operators = None, is_helical_symmetry = None): self.shifted_used_ncs_info = ncs_info_object(file_name = file_name, number_of_operators = number_of_operators, is_helical_symmetry = is_helical_symmetry) def set_solvent_fraction(self, solvent_fraction): self.solvent_fraction = solvent_fraction def set_n_residues(self, n_residues): # may not be the same as seq file self.n_residues = n_residues def set_output_ncs_au_map_info(self, file_name = None, crystal_symmetry = None, origin = None, all = None): self.output_ncs_au_map_info = map_info_object(file_name = file_name, crystal_symmetry = crystal_symmetry, origin = origin, all = all, is_map = True) def set_output_ncs_au_mask_info(self, file_name = None, crystal_symmetry = None, origin = None, all = None): self.output_ncs_au_mask_info = map_info_object(file_name = file_name, crystal_symmetry = crystal_symmetry, origin = origin, all = all, is_map = False) def set_output_ncs_au_pdb_info(self, file_name = None, n_residues = None): self.output_ncs_au_pdb_info = pdb_info_object(file_name = file_name, n_residues = n_residues) def set_output_box_map_info(self, file_name = None, crystal_symmetry = None, origin = None, all = None): self.output_box_map_info = map_info_object(file_name = file_name, crystal_symmetry = crystal_symmetry, origin = origin, all = all, is_map = True) def set_output_box_mask_info(self, file_name = None, crystal_symmetry = None, origin = None, all = None): self.output_box_mask_info = map_info_object(file_name = file_name, crystal_symmetry = crystal_symmetry, origin = origin, all = all, is_map = False) def add_output_region_map_info(self, file_name = None, crystal_symmetry = None, origin = None, all = None, map_id = None): self.output_region_map_info_list.append(map_info_object( file_name = file_name, crystal_symmetry = crystal_symmetry, origin = origin, all = all, id = len(self.output_region_map_info_list)+1, map_id = map_id, is_map = True) ) def add_output_region_pdb_info(self, file_name = None, n_residues = None): self.output_region_pdb_info_list.append(pdb_info_object( file_name = file_name, n_residues = n_residues) ) def show_summary(self, out = sys.stdout): print("\n ========== Summary of %s: ======== \n" %(self.object_type), file = out) print("Created: %s" %(self.init_asctime), file = out) print("\nInput files used:\n", file = out) if self.input_map_info: self.input_map_info.show_summary(out = out) if self.input_pdb_info: self.input_pdb_info.show_summary(out = out) if self.input_ncs_info: self.input_ncs_info.show_summary(out = out) if self.input_seq_info: self.input_seq_info.show_summary(out = out) print(file = out) if self.crystal_symmetry: print("Working unit cell: %.1f %.1f %.1f %.1f %.1f %.1f " %( self.crystal_symmetry.unit_cell().parameters()), file = out) if self.n_residues: print("Estimated total number of residues: %d" %(self.n_residues), file = out) if self.solvent_fraction: print("Estimated solvent fraction: %5.3f" %(self.solvent_fraction), file = out) if self.origin_shift and self.origin_shift != (0, 0, 0): print("\nOrigin offset applied: %.1f %.1f %.1f" %(self.origin_shift), file = out) else: print("\nNo origin offset applied", file = out) if self.shifted_map_info: print("\nShifted/sharpened map, pdb and ncs files created "+\ "(after origin offset):\n", file = out) if self.shifted_map_info: self.shifted_map_info.show_summary(out = out) if self.shifted_pdb_info: self.shifted_pdb_info.show_summary(out = out) if self.shifted_ncs_info: self.shifted_ncs_info.show_summary(out = out) if self.output_ncs_au_pdb_info: print("\nOutput PDB file with dummy atoms representing the NCS AU:", file = out) self.output_ncs_au_pdb_info.show_summary(out = out) if self.output_ncs_au_mask_info or self.output_ncs_au_map_info: print("\nOutput map files showing just the NCS AU (same size", end = ' ', file = out) if self.origin_shift and self.origin_shift != (0, 0, 0): print("\nand location as shifted map files:\n", file = out) else: print("\nand location as input map:\n", file = out) if self.output_ncs_au_mask_info: self.output_ncs_au_mask_info.show_summary(out = out) if self.output_ncs_au_map_info: self.output_ncs_au_map_info.show_summary(out = out) if self.output_box_mask_info or self.output_box_map_info: print("\nOutput cut-out map files trimmed to contain just "+\ "the \nNCS AU (superimposed on", end = ' ', file = out) if self.origin_shift and self.origin_shift != (0, 0, 0): print("shifted map files, note origin offset):\n", file = out) else: print("input map, note origin offset):\n", file = out) if self.output_box_mask_info: self.output_box_mask_info.show_summary(out = out) if self.output_box_map_info: self.output_box_map_info.show_summary(out = out) if self.output_region_pdb_info_list: print("\nOutput PDB files representing one region of connected"+\ " density.\nThese are useful for marking where to look in cut-out map"+\ " files.", file = out) for output_region_pdb_info in self.output_region_pdb_info_list: output_region_pdb_info.show_summary(out = out) if self.output_region_map_info_list: print("\nOutput cut-out map files trimmed to contain just "+\ "one region of \nconnected density (superimposed on", end = ' ', file = out) if self.origin_shift and self.origin_shift != (0, 0, 0): print("shifted map files, note origin offset):\n", file = out) else: print(" input map, note origin offset):\n", file = out) for output_region_map_info in self.output_region_map_info_list: output_region_map_info.show_summary(out = out) print("\n"+50*"="+"\n", file = out) class make_ccp4_map: # just a holder so map_to_structure_factors will run # XXX Replace with map_manager def __init__(self, map = None, unit_cell = None): self.data = map self.unit_cell_parameters = unit_cell.parameters() self.space_group_number = 1 self.unit_cell_grid = map.all() def unit_cell(self): return self.crystal_symmetry().unit_cell() def unit_cell_crystal_symmetry(self): return self.crystal_symmetry() def map_data(self): return self.data def crystal_symmetry(self): return crystal.symmetry(self.unit_cell_parameters, self.space_group_number) class b_vs_region_info: def __init__(self): self.b_iso = 0. self.b_vs_region_dict = {} self.sa_sum_v_vs_region_dict = {} self.sa_nn_vs_region_dict = {} self.sa_ratio_b_vs_region_dict = {} class box_sharpening_info: def __init__(self, tracking_data = None, crystal_symmetry = None, solvent_fraction = None, b_iso = None, resolution = None, d_min_ratio = None, scale_max = None, lower_bounds = None, upper_bounds = None, wrapping = None, n_real = None, n_buffer = None, map_data = None, smoothing_radius = None, smoothed_box_mask_data = None, original_box_map_data = None, ): from libtbx import adopt_init_args adopt_init_args(self, locals()) del self.tracking_data # do not save it if tracking_data: self.crystal_symmetry = tracking_data.crystal_symmetry self.solvent_fraction = tracking_data.solvent_fraction self.wrapping = tracking_data.params.crystal_info.use_sg_symmetry def get_gaussian_weighting(self, out = sys.stdout): # return a gaussian function centered on center of the map, fall-off # based on smoothing_radius # Calculate weight map, max near location of centers_ncs_cart # U = rmsd**2 # (b_eff = 8*3.14159**2*U) # rmsd is at least distance between centers, not too much bigger than # unit cell size, typically 10-20 A, print("\nFall-off of local weight is 1/%6.1f A\n" %( self.smoothing_radius), file = out) u = self.smoothing_radius**2 from cctbx import xray xrs, scatterers = set_up_xrs(crystal_symmetry = self.crystal_symmetry) unit_cell = self.crystal_symmetry.unit_cell() for xyz_fract in [(0.5, 0.5, 0.5, )]: scatterers.append( xray.scatterer(scattering_type = "H", label = "H", site = xyz_fract, u = u, occupancy = 1.0)) xrs = xray.structure(xrs, scatterers = scatterers) f_array, phases = get_f_phases_from_map(map_data = self.map_data, crystal_symmetry = self.crystal_symmetry, d_min = self.resolution, scale_max = self.scale_max, d_min_ratio = self.d_min_ratio, get_remove_aniso_object = False, # don't need it out = out) weight_f_array = f_array.structure_factors_from_scatterers( algorithm = 'direct', xray_structure = xrs).f_calc() weight_map = get_map_from_map_coeffs(map_coeffs = weight_f_array, crystal_symmetry = self.crystal_symmetry, n_real = self.map_data.all()) min_value = weight_map.as_1d().min_max_mean().min weight_map = weight_map-min_value # all positive or zero max_value = weight_map.as_1d().min_max_mean().max weight_map = weight_map/max(1.e-10, max_value) # normalize; max = 1 now min_value = 1.e-10 # just a small value for all distances far from center s = (weight_map = 0: mid = (1+sum)//2 else: mid = (-1+sum)//2 alb = min(mid-minimum, lb+self.n_buffer) aub = max(mid+minimum, ub-self.n_buffer) adjusted_lower_bounds.append(alb) adjusted_upper_bounds.append(aub) delta_lower_bounds.append(alb-lb) delta_upper_bounds.append(aub-ub) return adjusted_lower_bounds, adjusted_upper_bounds, \ delta_lower_bounds, delta_upper_bounds def merge_into_overall_map(self, overall_map = None): # Smoothly fill out edges of the small map with overall_map assert self.smoothed_box_mask_data is not None assert self.original_box_map_data is not None self.map_data = (self.map_data * self.smoothed_box_mask_data) + \ (self.original_box_map_data * (1-self.smoothed_box_mask_data)) def remove_buffer(self, out = sys.stdout): # remove the buffer from this box new_lower_bounds, new_upper_bounds, delta_lower, delta_upper = \ self.remove_buffer_from_bounds() cut_out_lower_bounds = [] cut_out_upper_bounds = [] for o, a, dlb, dub in zip(self.map_data.origin(), self.map_data.all(), delta_lower, delta_upper): cut_out_lower_bounds.append(o+dlb) cut_out_upper_bounds.append(a+dub-1) self.map_data, self.crystal_symmetry, \ self.smoothed_box_mask_data, self.original_box_map_data = cut_out_map( map_data = self.map_data, crystal_symmetry = self.crystal_symmetry, soft_mask = False, resolution = self.resolution, shift_origin = True, min_point = cut_out_lower_bounds, max_point = cut_out_upper_bounds, out = out) self.lower_bounds = new_lower_bounds self.upper_bounds = new_upper_bounds class sharpening_info: def __init__(self, tracking_data = None, crystal_symmetry = None, is_crystal = None, sharpening_method = None, solvent_fraction = None, n_residues = None, ncs_copies = None, ncs_file = None, seq_file = None, sequence = None, n_real = None, region_weight = None, n_bins = None, eps = None, d_min = None, d_min_ratio = None, scale_max = None, input_d_cut = None, b_blur_hires = None, rmsd = None, rmsd_resolution_factor = None, k_sol = None, b_sol = None, fraction_complete = None, wrapping = None, sharpening_target = None, residual_target = None, fraction_occupied = None, nproc = None, multiprocessing = None, queue_run_command = None, resolution = None, # changed from d_cut resolution_dependent_b = None, # linear sharpening normalize_amplitudes_in_resdep = None, # linear sharpening b_sharpen = None, b_iso = None, # expected B_iso after applying b_sharpen k_sharpen = None, optimize_b_blur_hires = None, iterate = None, optimize_d_cut = None, kurtosis = None, adjusted_sa = None, sa_ratio = None, normalized_regions = None, score = None, input_weight_map_pickle_file = None, output_weight_map_pickle_file = None, read_sharpened_maps = None, write_sharpened_maps = None, select_sharpened_map = None, output_directory = None, smoothing_radius = None, local_sharpening = None, local_aniso_in_local_sharpening = None, overall_before_local = None, use_local_aniso = None, original_aniso_obj = None, auto_sharpen = None, box_in_auto_sharpen = None, density_select_in_auto_sharpen = None, density_select_threshold_in_auto_sharpen = None, use_weak_density = None, discard_if_worse = None, max_box_fraction = None, cc_cut = None, max_cc_for_rescale = None, scale_using_last = None, density_select_max_box_fraction = None, mask_atoms = None, mask_atoms_atom_radius = None, value_outside_atoms = None, soft_mask = None, allow_box_if_b_iso_set = None, search_b_min = None, search_b_max = None, search_b_n = None, adjust_region_weight = None, region_weight_method = None, region_weight_factor = None, region_weight_buffer = None, region_weight_default = None, target_b_iso_ratio = None, signal_min = None, target_b_iso_model_scale = None, box_sharpening_info_obj = None, chain_type = None, target_scale_factors = None, remove_aniso = None, d_min_list = None, verbose = None, resolve_size = None, pdb_inp = None, # XXX probably do not need this local_solvent_fraction = None, wang_radius = None, buffer_radius = None, pseudo_likelihood = None, preliminary_sharpening_done = False, adjusted_path_length = None, ): from libtbx import adopt_init_args adopt_init_args(self, locals()) del self.tracking_data # don't need it as part of the object del self.box_sharpening_info_obj# don't need it as part of the object del self.pdb_inp # don't need it as part of the object if tracking_data: # use tracking data information self.update_with_tracking_data(tracking_data = tracking_data) if box_sharpening_info_obj: # update information self.update_with_box_sharpening_info( box_sharpening_info_obj = box_sharpening_info_obj) if self.resolution_dependent_b is None: self.resolution_dependent_b = [0, 0, 0] if self.target_scale_factors and \ self.sharpening_method!= 'model_sharpening' \ and self.sharpening_method!= 'half_map_sharpening': assert self.sharpening_method is None # XXX may want to print out error self.sharpening_method = 'model_sharpening' if self.sharpening_method == 'b_iso' and self.k_sharpen is not None: self.k_sharpen = None if pdb_inp: self.sharpening_method = 'model_sharpening' self.box_in_auto_sharpen = True self.density_select_in_auto_sharpen = False self.sharpening_target = 'model' def get_d_cut(self): if self.input_d_cut is not None: return self.input_d_cut else: return self.resolution def get_target_b_iso(self): if self.target_b_iso_ratio is None: return None if self.resolution is None: return None return self.target_b_iso_ratio*self.resolution**2 def set_resolution_dependent_b(self, resolution_dependent_b = None, sharpening_method = 'resolution_dependent'): if resolution_dependent_b: self.resolution_dependent_b = resolution_dependent_b if sharpening_method: self.sharpening_method = sharpening_method def sharpening_is_defined(self): if self.sharpening_method is None: return False if self.target_scale_factors: return True if self.sharpening_method == 'target_b_iso_to_d_cut': return True if self.b_iso is not None or \ self.b_sharpen is not None or \ (self.resolution_dependent_b is not None and self.resolution_dependent_b!= [0, 0, 0]): return True return False def update_with_box_sharpening_info(self, box_sharpening_info_obj = None): if not box_sharpening_info_obj: return self self.crystal_symmetry = box_sharpening_info_obj.crystal_symmetry self.solvent_fraction = box_sharpening_info_obj.solvent_fraction self.wrapping = box_sharpening_info_obj.wrapping self.n_real = box_sharpening_info_obj.n_real return self def update_with_tracking_data(self, tracking_data = None): self.update_with_params(params = tracking_data.params, crystal_symmetry = tracking_data.crystal_symmetry, solvent_fraction = tracking_data.solvent_fraction, n_residues = tracking_data.n_residues, ncs_copies = tracking_data.input_ncs_info.number_of_operators) return self def update_with_params(self, params = None, crystal_symmetry = None, is_crystal = None, solvent_fraction = None, auto_sharpen = None, sharpening_method = None, pdb_inp = None, half_map_data_list = None, n_residues = None, ncs_copies = None): self.crystal_symmetry = crystal_symmetry self.is_crystal = is_crystal self.solvent_fraction = solvent_fraction self.auto_sharpen = auto_sharpen self.n_residues = n_residues self.ncs_copies = ncs_copies self.seq_file = params.input_files.seq_file self.chain_type = params.crystal_info.chain_type self.verbose = params.control.verbose self.resolve_size = params.control.resolve_size self.multiprocessing = params.control.multiprocessing self.nproc = params.control.nproc self.queue_run_command = params.control.queue_run_command self.wrapping = params.crystal_info.use_sg_symmetry self.fraction_occupied = params.map_modification.fraction_occupied self.sa_percent = params.map_modification.sa_percent self.region_weight = params.map_modification.region_weight self.max_regions_to_test = params.map_modification.max_regions_to_test self.regions_to_keep = params.map_modification.regions_to_keep self.d_min_ratio = params.map_modification.d_min_ratio self.scale_max = params.map_modification.scale_max self.input_d_cut = params.map_modification.input_d_cut self.b_blur_hires = params.map_modification.b_blur_hires self.rmsd = params.map_modification.rmsd self.rmsd_resolution_factor = params.map_modification.rmsd_resolution_factor self.k_sol = params.map_modification.k_sol self.b_sol = params.map_modification.b_sol self.fraction_complete = params.map_modification.fraction_complete self.resolution = params.crystal_info.resolution # changed from d_cut # NOTE: # resolution = X-ray resolution or nominal resolution of cryoEM map # high-res cutoff of reflections is d_min*d_min_ratio self.buffer_radius = params.crystal_info.buffer_radius self.wang_radius = params.crystal_info.wang_radius self.pseudo_likelihood = params.crystal_info.pseudo_likelihood self.max_box_fraction = params.map_modification.max_box_fraction self.cc_cut = params.map_modification.cc_cut self.max_cc_for_rescale = params.map_modification.max_cc_for_rescale self.scale_using_last = params.map_modification.scale_using_last self.density_select_max_box_fraction = params.map_modification.density_select_max_box_fraction self.mask_atoms = params.map_modification.mask_atoms self.mask_atoms_atom_radius = params.map_modification.mask_atoms_atom_radius self.value_outside_atoms = params.map_modification.value_outside_atoms self.soft_mask = params.map_modification.soft_mask self.allow_box_if_b_iso_set = params.map_modification.allow_box_if_b_iso_set self.k_sharpen = params.map_modification.k_sharpen self.optimize_b_blur_hires = params.map_modification.optimize_b_blur_hires self.iterate = params.map_modification.iterate self.optimize_d_cut = params.map_modification.optimize_d_cut self.sharpening_target = params.map_modification.sharpening_target self.residual_target = params.map_modification.residual_target self.eps = params.map_modification.eps self.n_bins = params.map_modification.n_bins self.input_weight_map_pickle_file = params.input_files.input_weight_map_pickle_file self.output_weight_map_pickle_file = params.output_files.output_weight_map_pickle_file self.read_sharpened_maps = params.map_modification.read_sharpened_maps self.write_sharpened_maps = params.map_modification.write_sharpened_maps self.select_sharpened_map = params.map_modification.select_sharpened_map self.output_directory = params.output_files.output_directory self.smoothing_radius = params.map_modification.smoothing_radius self.local_sharpening = params.map_modification.local_sharpening self.local_aniso_in_local_sharpening = \ params.map_modification.local_aniso_in_local_sharpening self.overall_before_local = \ params.map_modification.overall_before_local self.box_in_auto_sharpen = params.map_modification.box_in_auto_sharpen self.density_select_in_auto_sharpen = params.map_modification.density_select_in_auto_sharpen self.density_select_threshold_in_auto_sharpen = params.map_modification.density_select_threshold_in_auto_sharpen self.use_weak_density = params.map_modification.use_weak_density self.discard_if_worse = params.map_modification.discard_if_worse self.box_center = params.map_modification.box_center self.box_size = params.map_modification.box_size self.target_n_overlap = params.map_modification.target_n_overlap self.restrict_map_size = params.map_modification.restrict_map_size self.remove_aniso = params.map_modification.remove_aniso self.min_ratio_of_ncs_copy_to_first = \ params.segmentation.min_ratio_of_ncs_copy_to_first self.max_ratio_to_target = params.segmentation.max_ratio_to_target self.min_ratio_to_target = params.segmentation.min_ratio_to_target self.residues_per_region = params.segmentation.residues_per_region self.mask_padding_fraction = \ params.segmentation.mask_padding_fraction self.fraction_of_max_mask_threshold = \ params.segmentation.fraction_of_max_mask_threshold self.cell_cutoff_for_solvent_from_mask = \ params.segmentation.cell_cutoff_for_solvent_from_mask self.starting_density_threshold = \ params.segmentation.starting_density_threshold self.density_threshold = params.segmentation.density_threshold self.min_ratio = params.segmentation.min_ratio self.min_volume = params.segmentation.min_volume self.search_b_min = params.map_modification.search_b_min self.search_b_max = params.map_modification.search_b_max self.search_b_n = params.map_modification.search_b_n self.adjust_region_weight = params.map_modification.adjust_region_weight self.region_weight_method = params.map_modification.region_weight_method self.region_weight_factor = params.map_modification.region_weight_factor self.region_weight_buffer = params.map_modification.region_weight_buffer self.region_weight_default = params.map_modification.region_weight_default self.target_b_iso_ratio = params.map_modification.target_b_iso_ratio self.signal_min = params.map_modification.signal_min self.target_b_iso_model_scale = params.map_modification.target_b_iso_model_scale if sharpening_method is not None: self.sharpening_method = sharpening_method if not self.sharpening_method and \ len(params.map_modification.auto_sharpen_methods) == 1: self.sharpening_method = params.map_modification.auto_sharpen_methods[0] if half_map_data_list or self.sharpening_method == 'half_map_sharpening': self.sharpening_method = 'half_map_sharpening' self.sharpening_target = 'half_map' elif pdb_inp or self.sharpening_method == 'model_sharpening': self.sharpening_method = 'model_sharpening' self.box_in_auto_sharpen = True self.density_select_in_auto_sharpen = False self.sharpening_target = 'model' elif params.map_modification.b_iso is not None or \ params.map_modification.b_sharpen is not None: if self.sharpening_method is None: raise Sorry("b_iso is not set") # if sharpening values are specified, set them if params.map_modification.b_iso is not None: self.b_iso = params.map_modification.b_iso # but we need b_sharpen elif params.map_modification.b_sharpen is not None: self.b_sharpen = params.map_modification.b_sharpen elif (params.map_modification.resolution_dependent_b is not None and params.map_modification.resolution_dependent_b!= [0, 0, 0]): self.sharpening_method = 'resolution_dependent' self.resolution_dependent_b = \ params.map_modification.resolution_dependent_b if self.sharpening_method == 'b_iso' and self.k_sharpen is not None: self.k_sharpen = None return self def show_summary(self, verbose = False, out = sys.stdout): method_summary_dict = { 'b_iso':"Overall b_iso sharpening", 'b_iso_to_d_cut':"b_iso sharpening to high_resolution cutoff", 'resolution_dependent':"Resolution-dependent sharpening", 'model_sharpening':"Model sharpening", 'half_map_sharpening':"Half-map sharpening", 'no_sharpening':"No sharpening", None:"No sharpening", } target_summary_dict = { 'adjusted_sa':"Adjusted surface area", 'adjusted_path_length':"Adjusted path length", 'kurtosis':"Map kurtosis", 'model':"Map-model CC", } print("\nSummary of sharpening:\n", file = out) if not hasattr(self, 'sharpening_method'): return # backwards compatibility print("Sharpening method used: %s\n" %( method_summary_dict.get(self.sharpening_method)), file = out) if self.sharpening_method == "b_iso": if self.b_sharpen is not None: print("Overall b_sharpen applied: %7.2f A**2" %( self.b_sharpen), file = out) if self.b_iso is not None: print("Final b_iso obtained: %7.2f A**2" %(self.b_iso), file = out) elif self.sharpening_method == "b_iso_to_d_cut": if self.b_sharpen is not None: print("Overall b_sharpen applied: %7.2f A**2" %( self.b_sharpen), file = out) if self.b_iso is not None: print("Final b_iso obtained: %7.2f A**2" %(self.b_iso), file = out) if self.input_d_cut: print("High-resolution cutoff: %7.2f A" %(self.input_d_cut), file = out) else: print("High-resolution cutoff: %7.2f A" %(self.resolution), file = out) elif self.sharpening_method == "resolution_dependent": print("Resolution-dependent b values (%7.2f, %7.2f, %7.2f)\n" %( tuple(self.resolution_dependent_b)), file = out) print("Effective b_iso vs resolution obtained:", file = out) from cctbx.maptbx.refine_sharpening import get_effective_b_values d_min_values, b_values = get_effective_b_values( d_min_ratio = self.d_min_ratio, resolution_dependent_b = self.resolution_dependent_b, resolution = self.resolution) print(" Resolution Effective B-iso", file = out) print(" (A) (A**2)", file = out) for dd, b in zip(d_min_values, b_values): print(" %7.1f %7.2f " %( dd, b), file = out) elif self.sharpening_method == "model_sharpening": print("Resolution-dependent model sharpening", file = out) if self.d_min_list and self.target_scale_factors: print("Scale vs resolution:", file = out) for d_min, sc in zip( self.d_min_list, self.target_scale_factors): print("Dmin: %7.2f Scale: %9.6f" %(d_min, sc), file = out) elif self.sharpening_method == "half_map_sharpening": print("Resolution-dependent half-map sharpening", file = out) if self.d_min_list and self.target_scale_factors: print("Scale vs resolution:", file = out) for d_min, sc in zip( self.d_min_list, self.target_scale_factors): print("Dmin: %7.2f Scale: %9.6f" %(d_min, sc), file = out) if self.sharpening_method in ["b_iso_to_d_cut"] and \ self.k_sharpen and self.resolution: print("Transition from sharpening"+\ " to not sharpening (k_sharpen):%7.2f " %(self.k_sharpen), file = out) print("\nSharpening target used: %s" %( target_summary_dict.get(self.sharpening_target)), file = out) if self.adjusted_sa is not None: print("Final adjusted map surface area: %7.2f" %(self.adjusted_sa), file = out) if self.kurtosis is not None: print("Final map kurtosis: %7.2f" %(self.kurtosis), file = out) if hasattr(self, 'adjusted_path_length') and \ self.adjusted_path_length is not None: print("Final adjusted path length: %7.2f A" %( self.adjusted_path_length), file = out) print(file = out) if verbose: for x in dir(self): if x.startswith("__"): continue if type(getattr(self, x)) in [type('a'), type(1), type(1.), type([]), type((1, 2, ))]: print("%s : %s" %(x, getattr(self, x)), file = out) def get_effective_b_iso(self, map_data = None, out = sys.stdout): map_coeffs_ra, map_coeffs, f_array, phases = effective_b_iso( map_data = map_data, resolution = self.resolution, d_min_ratio = self.d_min_ratio, scale_max = self.scale_max, crystal_symmetry = self.crystal_symmetry, out = out) return map_coeffs_ra.b_iso def sharpen_and_score_map(self, map_data = None, set_b_iso = False, out = sys.stdout): if self.n_real is None: # need to get it self.n_real = map_data.all() map_and_b = sharpen_map_with_si( sharpening_info_obj = self, map_data = map_data, resolution = self.resolution, out = out) self.map_data = map_and_b.map_data if set_b_iso: self.b_iso = map_and_b.final_b_iso score_map(map_data = self.map_data, sharpening_info_obj = self, out = null_out()) return self def show_score(self, out = sys.stdout): print("Adjusted surface area: %7.3f Kurtosis: %7.3f Score: %7.3f\n" %( self.adjusted_sa, self.kurtosis, self.score), file = out) def is_target_b_iso_to_d_cut(self): if self.sharpening_method == 'target_b_iso_to_d_cut': return True else: return False def is_b_iso_sharpening(self): if self.is_resolution_dependent_sharpening(): return False if self.is_model_sharpening(): return False if self.is_half_map_sharpening(): return False if self.is_external_map_sharpening(): return False return True def is_resolution_dependent_sharpening(self): if self.sharpening_method == 'resolution_dependent': return True else: return False def is_model_sharpening(self): if self.sharpening_method == 'model_sharpening': return True else: return False def is_external_map_sharpening(self): if self.sharpening_method == 'external_map_sharpening': return True else: return False def is_half_map_sharpening(self): if self.sharpening_method == 'half_map_sharpening': return True else: return False def as_map_coeffs(self, out = sys.stdout): map_data = getattr(self, 'map_data', None) if map_data: map_coeffs, dummy = get_f_phases_from_map(map_data = self.map_data, crystal_symmetry = self.crystal_symmetry, d_min = self.resolution, d_min_ratio = self.d_min_ratio, scale_max = self.scale_max, return_as_map_coeffs = True, out = out) return map_coeffs else: return None def as_map_data(self): return getattr(self, 'map_data', None) class ncs_group_object: def __init__(self, ncs_obj = None, ncs_ops_used = None, ncs_group_list = None, edited_mask = None, crystal_symmetry = None, max_cell_dim = None, origin_shift = None, edited_volume_list = None, region_range_dict = None, selected_regions = None, ncs_related_regions = None, self_and_ncs_related_regions = None, equiv_dict = None, map_files_written = None, bad_region_list = None, region_centroid_dict = None, region_scattered_points_dict = None, shared_group_dict = None, co = None, min_b = None, max_b = None, original_id_from_id = None, remainder_id_dict = None, # dict relating regions in a remainder object to # those in the original map ): if not selected_regions: selected_regions = [] if not ncs_related_regions: ncs_related_regions = [] if not self_and_ncs_related_regions: self_and_ncs_related_regions = [] if not map_files_written: map_files_written = [] if not max_cell_dim: max_cell_dim = 0. from libtbx import adopt_init_args adopt_init_args(self, locals()) if self.crystal_symmetry and not self.max_cell_dim: self.max_cell_dim = 0. for x in self.crystal_symmetry.unit_cell().parameters()[:3]: self.max_cell_dim = max(max_cell_dim, x) def as_info_object(self): return info_object( ncs_obj = self.ncs_obj, max_b = self.max_b, min_b = self.min_b, ncs_group_list = self.ncs_group_list, origin_shift = self.origin_shift, edited_volume_list = self.edited_volume_list, region_range_dict = self.region_range_dict, selected_regions = self.selected_regions, ncs_related_regions = self.ncs_related_regions, self_and_ncs_related_regions = self.self_and_ncs_related_regions, bad_region_list = self.bad_region_list, region_centroid_dict = self.region_centroid_dict, original_id_from_id = self.original_id_from_id, map_files_written = self.map_files_written, ) def set_ncs_ops_used(self, ncs_ops_used): self.ncs_ops_used = deepcopy(ncs_ops_used) def set_selected_regions(self, selected_regions): self.selected_regions = deepcopy(selected_regions) def set_ncs_related_regions(self, ncs_related_regions): self.ncs_related_regions = deepcopy(ncs_related_regions) def set_self_and_ncs_related_regions(self, self_and_ncs_related_regions): self.self_and_ncs_related_regions = deepcopy(self_and_ncs_related_regions) def set_map_files_written(self, map_files_written): self.map_files_written = deepcopy(map_files_written) def zero_if_none(x): if not x: return 0 else: return x def scale_map(map, scale_rms = 1.0, out = sys.stdout): sd = map.as_double().as_1d().sample_standard_deviation() if (sd > 1.e-10): scale = scale_rms/sd if 0: print("Scaling map by %7.3f to set SD = 1" %(scale), file = out) map = map*scale else: print("Cannot scale map...all zeros", file = out) return map def scale_map_coeffs(map_coeffs, scale_max = None, out = sys.stdout): f_array, phases = map_coeffs_as_fp_phi(map_coeffs) max_value = f_array.data().min_max_mean().max if scale_max and max_value is not None: scale = scale_max/max(1.e-10, max_value) else: scale = 1.0 if 0: print("Scaling map_coeffs by %9.3f to yield maximum of %7.0f" %( scale, scale_max), file = out) return f_array.array(data = f_array.data()*scale ).phase_transfer(phase_source = phases, deg = True) def get_map_object(file_name = None, must_allow_sharpening = None, get_map_labels = None, out = sys.stdout): # read a ccp4 map file and return sg, cell and map objects 2012-01-16 if not os.path.isfile(file_name): raise Sorry("The map file %s is missing..." %(file_name)) map_labels = None if file_name.endswith(".xplor"): import iotbx.xplor.map m = iotbx.xplor.map.reader(file_name = file_name) m.unit_cell_grid = m.map_data().all() # just so we have something m.space_group_number = 0 # so we have something else: from iotbx import mrcfile m = mrcfile.map_reader(file_name = file_name) print("MIN MAX MEAN RMS of map: %7.2f %7.2f %7.2f %7.2f " %( m.header_min, m.header_max, m.header_mean, m.header_rms), file = out) print("grid: ", m.unit_cell_grid, file = out) print("cell: %8.3f %8.3f %8.3f %8.3f %8.3f %8.3f " %tuple( m.unit_cell().parameters()), file = out) print("SG: ", m.unit_cell_crystal_symmetry().space_group_number(), file = out) if must_allow_sharpening and m.cannot_be_sharpened(): raise Sorry("Input map is already modified and should not be sharpened") if get_map_labels: map_labels = m.labels print("ORIGIN: ", m.map_data().origin(), file = out) print("EXTENT: ", m.map_data().all(), file = out) print("IS PADDED: ", m.map_data().is_padded(), file = out) map_data = m.data acc = map_data.accessor() shift_needed = not \ (map_data.focus_size_1d() > 0 and map_data.nd() == 3 and map_data.is_0_based()) if(shift_needed): map_data = map_data.shift_origin() origin_shift = ( m.map_data().origin()[0]/m.map_data().all()[0], m.map_data().origin()[1]/m.map_data().all()[1], m.map_data().origin()[2]/m.map_data().all()[2]) origin_frac = origin_shift # NOTE: fraction of NEW cell else: origin_frac = (0., 0., 0.) # determine if we need to trim off the outer part of the map duplicating inner offsets = [] need_offset = False for g, e in zip(m.unit_cell_grid, map_data.all() ): offset = e-g offsets.append(offset) if offsets == [1, 1, 1]: if origin_frac!= (0., 0., 0.): # this was a shifted map...we can't do this raise Sorry("Sorry if a CCP4 map has an origin other than (0, 0, 0) "+ "the extent \nof the map must be the same as the grid or 1 "+ "\ngreater for "+ "segment_and_split_map routines."+ "The file %s has a grid of %s and extent of %s" %( file_name, str(m.unit_cell_grid), str(map_data.all()))) map = map_data[:-1, :-1, :-1] acc = map.accessor() else: map = map_data # now get space group and cell from cctbx import crystal from cctbx import sgtbx if m.unit_cell_crystal_symmetry().space_group_number() == 0: n = 1 # fix mrc formatting else: n = m.unit_cell_crystal_symmetry().space_group_number() if hasattr(m, 'crystal_symmetry'): space_group_info = sgtbx.space_group_info(number = n) unit_cell = m.unit_cell_crystal_symmetry().unit_cell() original_unit_cell_grid = m.unit_cell_grid original_crystal_symmetry = crystal.symmetry( unit_cell = unit_cell, space_group_info = space_group_info) if original_crystal_symmetry and map.all() == m.unit_cell_grid: crystal_symmetry = original_crystal_symmetry print("\nUnit cell crystal symmetry used: ", file = out) else: crystal_symmetry = m.crystal_symmetry() print("\nBox crystal symmetry used: ", file = out) crystal_symmetry.show_summary(f = out) space_group = crystal_symmetry.space_group() unit_cell = crystal_symmetry.unit_cell() else: space_group = None unit_cell = None crystal_symmetry = None original_crystal_symmetry, original_unit_cell_grid = None, None map = scale_map(map, out = out) if get_map_labels: return map, space_group, unit_cell, crystal_symmetry, origin_frac, acc, \ original_crystal_symmetry, original_unit_cell_grid, map_labels else: return map, space_group, unit_cell, crystal_symmetry, origin_frac, acc, \ original_crystal_symmetry, original_unit_cell_grid def write_ccp4_map(crystal_symmetry, file_name, map_data, output_unit_cell_grid = None, labels = None): if output_unit_cell_grid is None: output_unit_cell_grid = map_data.all() if labels is None: labels = flex.std_string([""]) iotbx.mrcfile.write_ccp4_map( file_name = file_name, unit_cell = crystal_symmetry.unit_cell(), space_group = crystal_symmetry.space_group(), unit_cell_grid = output_unit_cell_grid, map_data = map_data.as_double(), labels = labels) def set_up_xrs(crystal_symmetry = None): # dummy xrs to write out atoms lines = ["ATOM 92 SG CYS A 10 8.470 28.863 18.423 1.00 22.05 S"] # just a random line to set up x-ray structure from cctbx.array_family import flex pdb_inp = iotbx.pdb.input(source_info = "", lines = lines) xrs = pdb_inp.xray_structure_simple(crystal_symmetry = crystal_symmetry) scatterers = flex.xray_scatterer() return xrs, scatterers def write_atoms(tracking_data = None, sites = None, file_name = None, crystal_symmetry = None, atom_name = None, resname = None, atom_type = None, occ = None, out = sys.stdout): if crystal_symmetry is None: crystal_symmetry = tracking_data.crystal_symmetry xrs, scatterers = set_up_xrs(crystal_symmetry = crystal_symmetry) from cctbx import xray unit_cell = crystal_symmetry.unit_cell() for xyz_cart in sites: scatterers.append( xray.scatterer(scattering_type = "O", label = "O", site = unit_cell.fractionalize(xyz_cart), u = 0.38, occupancy = 1.0)) text = write_xrs(xrs = xrs, scatterers = scatterers, file_name = file_name, out = out) if atom_name and resname and atom_type: text = text.replace("O O ", " %2s %3s A" %(atom_name, resname) ) text = text.replace(" O", " %1s" %(atom_type)) if occ: text = text.replace(" 1.00 ", " %.2f " %(occ)) return text def write_xrs(xrs = None, scatterers = None, file_name = "atoms.pdb", out = sys.stdout): from cctbx import xray xrs = xray.structure(xrs, scatterers = scatterers) text = xrs.as_pdb_file() if file_name: f = open(file_name, 'w') print(text, file = f) f.close() print("Atoms written to %s" %file_name, file = out) return text def get_b_iso(miller_array, d_min = None, return_aniso_scale_and_b = False, d_max = 100000.): if d_min: res_cut_array = miller_array.resolution_filter(d_max = d_max, d_min = d_min) else: res_cut_array = miller_array from mmtbx.scaling import absolute_scaling try: aniso_scale_and_b = absolute_scaling.ml_aniso_absolute_scaling( miller_array = res_cut_array, n_residues = 200, n_bases = 0, ignore_errors = True) b_cart = aniso_scale_and_b.b_cart except Exception as e: b_cart = [0, 0, 0] aniso_scale_and_b = None b_aniso_mean = 0. if b_cart: for k in [0, 1, 2]: b_aniso_mean+= b_cart[k] if return_aniso_scale_and_b: return b_aniso_mean/3.0, aniso_scale_and_b else: # usual return b_aniso_mean/3.0 def map_coeffs_as_fp_phi(map_coeffs): amplitudes = map_coeffs.amplitudes() amplitudes.set_observation_type_xray_amplitude() assert amplitudes.is_real_array() phases = map_coeffs.phases(deg = True) return amplitudes, phases def map_coeffs_to_fp(map_coeffs): amplitudes = map_coeffs.amplitudes() amplitudes.set_observation_type_xray_amplitude() assert amplitudes.is_real_array() return amplitudes def get_f_phases_from_model(f_array = None, pdb_inp = None, overall_b = None, k_sol = None, b_sol = None, out = sys.stdout): xray_structure = pdb_inp.construct_hierarchy().extract_xray_structure( crystal_symmetry = f_array.crystal_symmetry()) print("Getting map coeffs from model with %s atoms.." %( xray_structure.sites_frac().size()), file = out) if overall_b is not None: print("Setting overall b_iso to %7.1f for model " %( overall_b), file = out) xray_structure.set_b_iso(value = overall_b) model_f_array = f_array.structure_factors_from_scatterers( xray_structure = xray_structure).f_calc() return model_f_array def get_f_phases_from_map( map_data = None, crystal_symmetry = None, d_min = None, d_max = 100000., d_min_ratio = None, return_as_map_coeffs = False, remove_aniso = None, get_remove_aniso_object = True, scale_max = None, origin_frac = None, out = sys.stdout): if d_min is not None: d_min_use = d_min if d_min_ratio is not None: d_min_use = d_min*d_min_ratio else: d_min_use = None if map_data.origin() != (0,0,0): map_data = map_data.shift_origin() from iotbx.map_manager import map_manager mm = map_manager(map_data = map_data, unit_cell_grid = map_data.all(), unit_cell_crystal_symmetry = crystal_symmetry, wrapping = False) map_coeffs = mm.map_as_fourier_coefficients(d_min = d_min_use, d_max = d_max if d_min_use is not None else None) if map_coeffs.data().size() < 1: raise Sorry("No map coefficients found in calculation of map at "+ "resolution of %.2f A." %( d_min_use)) if origin_frac and tuple(origin_frac) != (0., 0., 0.): # shift origin map_coeffs = map_coeffs.translational_shift(origin_frac, deg = False) map_coeffs = scale_map_coeffs(map_coeffs, scale_max = scale_max, out = out) if remove_aniso: print("\nRemoving aniso in data before analysis\n", file = out) get_remove_aniso_object = True from cctbx.maptbx.refine_sharpening import analyze_aniso map_coeffs, map_coeffs_ra = analyze_aniso( remove_aniso = remove_aniso, get_remove_aniso_object = get_remove_aniso_object, map_coeffs = map_coeffs, resolution = d_min, out = out) if return_as_map_coeffs: return map_coeffs, map_coeffs_ra else: return map_coeffs_as_fp_phi(map_coeffs) def apply_sharpening(map_coeffs = None, sharpening_info_obj = None, n_real = None, b_sharpen = None, crystal_symmetry = None, target_scale_factors = None, f_array = None, phases = None, d_min = None, k_sharpen = None, b_blur_hires = None, include_sharpened_map_coeffs = False, out = sys.stdout): if map_coeffs and f_array is None and phases is None: f_array, phases = map_coeffs_as_fp_phi(map_coeffs) if sharpening_info_obj is not None: b_sharpen = sharpening_info_obj.b_sharpen b_blur_hires = sharpening_info_obj.b_blur_hires k_sharpen = sharpening_info_obj.k_sharpen if sharpening_info_obj.input_d_cut: d_min = sharpening_info_obj.input_d_cut else: d_min = sharpening_info_obj.resolution# changed from d_cut n_real = sharpening_info_obj.n_real target_scale_factors = sharpening_info_obj.target_scale_factors n_bins = sharpening_info_obj.n_bins remove_aniso = sharpening_info_obj.remove_aniso resolution = sharpening_info_obj.resolution if target_scale_factors: assert sharpening_info_obj is not None print("\nApplying target scale factors vs resolution", file = out) if not map_coeffs: map_coeffs = f_array.phase_transfer(phase_source = phases, deg = True) f_array, phases = map_coeffs_as_fp_phi(map_coeffs) f_array_b_iso = get_b_iso(f_array, d_min = d_min) if not f_array.binner(): (local_d_max, local_d_min) = f_array.d_max_min( d_max_is_highest_defined_if_infinite=True) f_array.setup_binner(n_bins = n_bins, d_max = local_d_max, d_min = local_d_min) from cctbx.maptbx.refine_sharpening import apply_target_scale_factors map_and_b = apply_target_scale_factors(f_array = f_array, phases = phases, resolution = d_min, target_scale_factors = target_scale_factors, n_real = n_real, out = out) return map_and_b elif b_sharpen is None or ( b_sharpen in [0, None] and k_sharpen in [0, None]): if not map_coeffs: map_coeffs = f_array.phase_transfer(phase_source = phases, deg = True) map_data = get_map_from_map_coeffs(map_coeffs = map_coeffs, crystal_symmetry = crystal_symmetry, n_real = n_real) return map_and_b_object(map_data = map_data) elif k_sharpen is None or d_min is None or k_sharpen<= 0 or \ ( b_blur_hires is None and b_sharpen < 0): # 2016-08-10 original method: apply b_sharpen to all data # Use this if blurring (b_sharpen<0) or if k_sharpen is not set from cctbx import adptbx # next lines from xtriage (basic_analysis.py) b_cart_aniso_removed = [ b_sharpen, b_sharpen, b_sharpen, 0, 0, 0] from mmtbx.scaling import absolute_scaling u_star_aniso_removed = adptbx.u_cart_as_u_star( f_array.unit_cell(), adptbx.b_as_u( b_cart_aniso_removed ) ) f_array_sharpened = absolute_scaling.anisotropic_correction( f_array, 0.0, u_star_aniso_removed, must_be_greater_than = -0.0001) else: # Apply sharpening only to data from infinity to d_min, with transition # steepness of k_sharpen. # 2017-08-21 if b_blur_hires is set, sharpen with # b_sharpen-b_blur_hires data beyond d_min (with same # transition, so transition goes from b_sharpen TO b_sharpen-b_blur_hires data_array = f_array.data() sthol_array = f_array.sin_theta_over_lambda_sq() d_spacings = f_array.d_spacings() scale_array = flex.double() import math if b_blur_hires is not None: b_sharpen_hires_use = b_sharpen-b_blur_hires else: b_sharpen_hires_use = 0. for x, (ind, sthol), (ind1, d) in zip(data_array, sthol_array, d_spacings): # for small value b = b_sharpen # for large value b = -b_sharpen_hires_use # transition is determined by k_sharpen value = min(20., max(-20., k_sharpen*(d_min-d))) lowres_weight = 1./(1.+math.exp(value)) hires_weight = max(0., 1-lowres_weight) b_sharpen_use = b_sharpen*lowres_weight+b_sharpen_hires_use*hires_weight log_scale = sthol*b_sharpen_use scale_array.append(math.exp(log_scale)) data_array = data_array*scale_array f_array_sharpened = f_array.customized_copy(data = data_array) actual_b_iso = get_b_iso(f_array_sharpened, d_min = d_min) print("B-iso after sharpening by b_sharpen = %6.1f is %7.2f\n" %( b_sharpen, actual_b_iso), file = out) sharpened_map_coeffs = f_array_sharpened.phase_transfer( phase_source = phases, deg = True) # And get new map map_data = get_map_from_map_coeffs(map_coeffs = sharpened_map_coeffs, crystal_symmetry = crystal_symmetry, n_real = n_real) mb = map_and_b_object(map_data = map_data, final_b_iso = actual_b_iso) if include_sharpened_map_coeffs: mb.sharpened_map_coeffs = sharpened_map_coeffs return mb def find_symmetry_center(map_data, crystal_symmetry = None, out = sys.stdout): # find center if necessary: origin = list(map_data.origin()) all = list(map_data.all()) centroid_wx = {} centroid_w = {} from cctbx import maptbx for ai in [0, 1, 2]: centroid_wx[ai] = 0. centroid_w[ai] = 0. for i in range(0, all[ai]): if ai == 0: start_tuple = tuple((i, 0, 0)) end_tuple = tuple((i, all[1]-1, all[2]-1)) #2019-11-05 not beyond na-1 elif ai == 1: start_tuple = tuple((0, i, 0)) end_tuple = tuple((all[0]-1, i, all[2]-1)) elif ai == 2: start_tuple = tuple((0, 0, i)) end_tuple = tuple((all[0]-1, all[1]-1, i)) new_map_data = maptbx.copy(map_data, start_tuple, end_tuple) mean_value = max(0., new_map_data.as_1d().as_double().min_max_mean().mean) centroid_wx[ai]+= mean_value*(i-origin[ai]) centroid_w[ai]+= mean_value if centroid_w[ai]>0: centroid_wx[ai] = centroid_wx[ai]/centroid_w[ai] print("CENTROID OF DENSITY: (%7.2f, %7.2f, %7.2f) (grid units) " %( tuple((centroid_wx[0], centroid_wx[1], centroid_wx[2], ))), file = out) xyz_fract = matrix.col((centroid_wx[0]/all[0], centroid_wx[1]/all[1], centroid_wx[2]/all[2], )) xyz_cart = crystal_symmetry.unit_cell().orthogonalize(xyz_fract) print("CENTROID (A): (%7.3f, %7.3f, %7.3f) " %( tuple(xyz_cart)), file = out) return xyz_cart def get_center_of_map(map_data, crystal_symmetry, place_on_grid_point = True): all = list(map_data.all()) origin = list(map_data.origin()) if place_on_grid_point: sx, sy, sz = [int(all[0]/2)+origin[0], int(all[1]/2)+origin[1], int(all[2]/2)+origin[2]] else: sx, sy, sz = [all[0]/2+origin[0], all[1]/2+origin[1], all[2]/2+origin[2]] site_fract = matrix.col((sx/all[0], sy/all[1], sz/all[2], )) return crystal_symmetry.unit_cell().orthogonalize(site_fract) def select_remaining_ncs_ops( map_data = None, crystal_symmetry = None, random_points = None, closest_sites = None, ncs_object = None, out = sys.stdout): # identify which NCS ops still apply. Choose the ones that maximize # scoring with score_ncs_in_map if ncs_object.max_operators()<1: return ncs_object used_ncs_id_list = [ncs_object.ncs_groups()[0].identity_op_id()] ncs_copies = ncs_object.max_operators() # find ncs_id that maximizes score (if any) improving = True from copy import deepcopy best_ops_to_keep = deepcopy(used_ncs_id_list) working_best_ops_to_keep = None best_score = None while improving: improving = False working_best_ops_to_keep = deepcopy(best_ops_to_keep) working_score = None for ncs_id in range(ncs_copies): if ncs_id in best_ops_to_keep:continue ops_to_keep = deepcopy(best_ops_to_keep) ops_to_keep.append(ncs_id) ncs_used_obj = ncs_object.deep_copy(ops_to_keep = ops_to_keep) score, ncs_cc = score_ncs_in_map(map_data = map_data, ncs_object = ncs_used_obj, ncs_in_cell_only = True, allow_score_with_pg = False, sites_orth = closest_sites, crystal_symmetry = crystal_symmetry, out = null_out()) if score is None: continue if working_score is None or score >working_score: working_score = score working_best_ops_to_keep = deepcopy(ops_to_keep) if working_score is not None and ( best_score is None or working_score>best_score): improving = True best_score = working_score best_ops_to_keep = deepcopy(working_best_ops_to_keep) ncs_used_obj = ncs_object.deep_copy(ops_to_keep = best_ops_to_keep) return ncs_used_obj def run_get_ncs_from_map(params = None, map_data = None, crystal_symmetry = None, map_symmetry_center = None, ncs_obj = None, fourier_filter = False, out = sys.stdout, ): # Get or check NCS operators. Try various possibilities for center of NCS # First Fourier filter map if resolution is set if fourier_filter and params.crystal_info.resolution: print("Fourier filtering at resolution of %.2f A" %( params.crystal_info.resolution), file = out) from iotbx.map_manager import map_manager mm = map_manager(map_data= map_data, unit_cell_crystal_symmetry = crystal_symmetry, unit_cell_grid = map_data.all(), wrapping=False) mm.resolution_filter(d_min=params.crystal_info.resolution) map_data = mm.map_data() ncs_obj_to_check = None if params.reconstruction_symmetry.symmetry and ( not ncs_obj or ncs_obj.max_operators()<2): if params.reconstruction_symmetry.optimize_center: center_try_list = [True, False] else: center_try_list = [True] elif ncs_obj: center_try_list = [True] ncs_obj_to_check = ncs_obj elif params.reconstruction_symmetry.optimize_center: center_try_list = [None] else: return None, None, None # did not even try # check separately for helical symmetry if params.reconstruction_symmetry.symmetry and \ params.reconstruction_symmetry.symmetry.lower() == 'helical': helical_list = [True] elif params.reconstruction_symmetry.symmetry and \ params.reconstruction_symmetry.symmetry.lower() in ['all', 'any'] and\ params.reconstruction_symmetry.include_helical_symmetry: helical_list = [False, True] else: helical_list = [False] new_ncs_obj, ncs_cc, ncs_score = None, None, None for use_center_of_map in center_try_list: for include_helical in helical_list: local_params = deepcopy(params) local_params.reconstruction_symmetry.include_helical_symmetry = \ include_helical new_ncs_obj, ncs_cc, ncs_score = get_ncs_from_map(params = local_params, map_data = map_data, map_symmetry_center = map_symmetry_center, use_center_of_map_as_center = use_center_of_map, crystal_symmetry = crystal_symmetry, ncs_obj_to_check = ncs_obj_to_check, out = out ) if new_ncs_obj: return new_ncs_obj, ncs_cc, ncs_score return new_ncs_obj, ncs_cc, ncs_score def get_ncs_from_map(params = None, map_data = None, map_symmetry_center = None, symmetry = None, symmetry_center = None, helical_rot_deg = None, helical_trans_z_angstrom = None, two_fold_along_x = None, op_max = None, crystal_symmetry = None, optimize_center = None, sites_orth = None, random_points = None, n_rescore = None, use_center_of_map_as_center = None, min_ncs_cc = None, identify_ncs_id = None, ncs_obj_to_check = None, ncs_in_cell_only = False, out = sys.stdout): # Purpose: check through standard point groups and helical symmetry to see # if map has symmetry. If symmetry == ANY then take highest symmetry that fits # Otherwise limit to the one specified with symmetry. # Use a library of symmetry matrices. For helical symmetry generate it # along the z axis. # Center of symmetry is as supplied, or center of map or center of density # If center is not supplied and use_center_of_map_as_center, try that # and return None if it fails to achieve a map cc of min_ncs_cc if ncs_in_cell_only is None: ncs_in_cell_only = (not params.crystal_info.use_sg_symmetry) if symmetry is None: symmetry = params.reconstruction_symmetry.symmetry if symmetry_center is None: symmetry_center = params.reconstruction_symmetry.symmetry_center if optimize_center is None: optimize_center = params.reconstruction_symmetry.optimize_center if helical_rot_deg is None: helical_rot_deg = params.reconstruction_symmetry.helical_rot_deg if helical_trans_z_angstrom is None: helical_trans_z_angstrom = \ params.reconstruction_symmetry.helical_trans_z_angstrom if n_rescore is None: n_rescore = params.reconstruction_symmetry.n_rescore if random_points is None: random_points = params.reconstruction_symmetry.random_points if op_max is None: op_max = params.reconstruction_symmetry.op_max if two_fold_along_x is None: two_fold_along_x = params.reconstruction_symmetry.two_fold_along_x if identify_ncs_id is None: identify_ncs_id = params.reconstruction_symmetry.identify_ncs_id if min_ncs_cc is None: min_ncs_cc = params.reconstruction_symmetry.min_ncs_cc # if ncs_obj_to_check is supplied...just use that ncs if ncs_obj_to_check and ncs_obj_to_check.max_operators()>1: symmetry = "SUPPLIED NCS" if map_symmetry_center is None: map_symmetry_center = get_center_of_map(map_data, crystal_symmetry) if optimize_center is None: if symmetry_center is None and (not use_center_of_map_as_center): optimize_center = True print("Setting optimize_center = True as no symmetry_center is supplied", file = out) else: optimize_center = False if symmetry_center is not None: symmetry_center = matrix.col(symmetry_center) elif use_center_of_map_as_center: print("Using center of map as NCS center", file = out) symmetry_center = map_symmetry_center else: # Find it if not ncs_obj_to_check: print("Finding NCS center as it is not supplied", file = out) symmetry_center = find_symmetry_center( map_data, crystal_symmetry = crystal_symmetry, out = out) print("Center of NCS (A): (%7.3f, %7.3f, %7.3f) " %( tuple(symmetry_center)), file = out) ncs_list = get_ncs_list(params = params, symmetry = symmetry, symmetry_center = symmetry_center, helical_rot_deg = helical_rot_deg, two_fold_along_x = two_fold_along_x, op_max = op_max, helical_trans_z_angstrom = helical_trans_z_angstrom, ncs_obj_to_check = ncs_obj_to_check, map_data = map_data, crystal_symmetry = crystal_symmetry, out = out, ) print("Total of %d NCS types to examine..." %(len(ncs_list)), file = out) if not sites_orth: sites_orth = get_points_in_map( map_data, n = random_points, crystal_symmetry = crystal_symmetry) # some random points in the map # Now make sure symmetry applied to points in points_list gives similar values results_list = [] for ncs_obj in ncs_list: symmetry = ncs_obj.get_ncs_name() score, cc_avg = score_ncs_in_map(map_data = map_data, ncs_object = ncs_obj, identify_ncs_id = identify_ncs_id, ncs_in_cell_only = ncs_in_cell_only, sites_orth = sites_orth, crystal_symmetry = crystal_symmetry, out = out) if cc_avg is None or cc_avg < min_ncs_cc: score = 0. # Do not allow low CC values to be used if score is None: print("symmetry:", symmetry, " no score", ncs_obj.max_operators(), file = out) else: results_list.append([score, cc_avg, ncs_obj, symmetry]) if not results_list: return None, None, None results_list.sort(key=itemgetter(0)) results_list.reverse() # Rescore top n_rescore if n_rescore and not ncs_obj_to_check: print("Rescoring top %d results" %(min(n_rescore, len(results_list))), file = out) rescore_list = results_list[n_rescore:] new_sites_orth = get_points_in_map( map_data, n = 10*random_points, crystal_symmetry = crystal_symmetry) new_sites_orth.extend(sites_orth) for orig_score, orig_cc_avg, ncs_obj, symmetry in results_list[:n_rescore]: score, cc_avg = score_ncs_in_map(map_data = map_data, ncs_object = ncs_obj, identify_ncs_id = identify_ncs_id, ncs_in_cell_only = ncs_in_cell_only, sites_orth = new_sites_orth, crystal_symmetry = crystal_symmetry, out = out) if cc_avg is None or cc_avg < min_ncs_cc: score = 0. # Do not allow low CC values to be used if score is None: print("symmetry:", symmetry, " no score", ncs_obj.max_operators(), file = out) else: rescore_list.append([score, cc_avg, ncs_obj, symmetry]) rescore_list.sort(key=itemgetter(0)) rescore_list.reverse() results_list = rescore_list if len(results_list) == 1: # check for C1 score, cc_avg, ncs_obj, symmetry = results_list[0] if symmetry and symmetry.strip() == 'C1': score = 1. cc_avg = 1. results_list = [[score, cc_avg, ncs_obj, symmetry], ] print("Ranking of NCS types:", file = out) if min_ncs_cc is not None: print("NOTE: any NCS type with CC < %.2f (min_ncs_cc) is unscored " %( min_ncs_cc), file = out) print("\n SCORE CC OPERATORS SYMMETRY", file = out) for score, cc_avg, ncs_obj, symmetry in results_list: if not symmetry: symmetry = "" if not cc_avg: cc_avg = 0.0 print(" %6.2f %5.2f %2d %s" %( score, cc_avg, ncs_obj.max_operators(), symmetry.strip(), ), file = out) score, cc_avg, ncs_obj, ncs_info = results_list[0] # Optimize center if necessary if optimize_center: symmetry_center, cc_avg, score, ncs_obj = optimize_center_position( map_data, sites_orth, crystal_symmetry, ncs_info, symmetry_center, ncs_obj, score, cc_avg, params = params, helical_rot_deg = helical_rot_deg, two_fold_along_x = two_fold_along_x, op_max = op_max, min_ncs_cc = min_ncs_cc, identify_ncs_id = identify_ncs_id, ncs_obj_to_check = ncs_obj_to_check, ncs_in_cell_only = ncs_in_cell_only, helical_trans_z_angstrom = helical_trans_z_angstrom, out = out) print("New center: (%7.3f, %7.3f, %7.3f)" %(tuple(symmetry_center)), file = out) if cc_avg < min_ncs_cc: print("No suitable symmetry found", file = out) return None, None, None print("\nBest NCS type is: ", end = ' ', file = out) print("\n SCORE CC OPERATORS SYMMETRY", file = out) if not ncs_info: ncs_info = "" print(" %6.2f %5.2f %2d %s Best NCS type" %( score, cc_avg, ncs_obj.max_operators(), ncs_info.strip(), ), file = out) return ncs_obj, cc_avg, score def optimize_center_position(map_data, sites_orth, crystal_symmetry, ncs_info, symmetry_center, ncs_obj, score, cc_avg, params = None, helical_rot_deg = None, two_fold_along_x = None, op_max = None, identify_ncs_id = None, ncs_obj_to_check = None, ncs_in_cell_only = None, min_ncs_cc = None, helical_trans_z_angstrom = None, out = sys.stdout): if ncs_info is None: ncs_info = "None" symmetry = ncs_info.split()[0] print("Optimizing center position...type is %s" %(ncs_info), file = out) if len(ncs_info.split())>1 and ncs_info.split()[1] == '(a)': two_fold_along_x = True elif len(ncs_info.split())>1 and ncs_info.split()[1] == '(b)': two_fold_along_x = False else: two_fold_along_x = None best_center = matrix.col(symmetry_center) best_ncs_obj = ncs_obj best_score = score best_cc_avg = cc_avg print("Starting center: (%7.3f, %7.3f, %7.3f)" %(tuple(best_center)), file = out) from libtbx.utils import null_out scale = 5. for itry in range(6): scale = scale/5. for i in range(-4, 5): for j in range(-4, 5): local_center = matrix.col(symmetry_center)+matrix.col((scale*i, scale*j, 0., )) ncs_list = get_ncs_list(params = params, symmetry = symmetry, symmetry_center = local_center, helical_rot_deg = helical_rot_deg, two_fold_along_x = two_fold_along_x, op_max = op_max, helical_trans_z_angstrom = helical_trans_z_angstrom, ncs_obj_to_check = ncs_obj_to_check, map_data = map_data, crystal_symmetry = crystal_symmetry, out = null_out(), ) if ncs_list: ncs_obj = ncs_list[0] score, cc_avg = score_ncs_in_map(map_data = map_data, ncs_object = ncs_obj, identify_ncs_id = identify_ncs_id, ncs_in_cell_only = ncs_in_cell_only, sites_orth = sites_orth, crystal_symmetry = crystal_symmetry, out = out) if cc_avg < min_ncs_cc: score = 0. # Do not allow low CC values to be used else: ncs_obj = None score, cc_avg = None, None if best_score is None or score>best_score: best_cc_avg = cc_avg best_score = score best_center = local_center best_ncs_obj = ncs_obj symmetry_center = best_center cc_avg = best_cc_avg score = best_score ncs_obj = best_ncs_obj return best_center, best_cc_avg, best_score, best_ncs_obj def score_ncs_in_map_point_group_symmetry( map_data = None, ncs_object = None, sites_orth = None, crystal_symmetry = None, out = sys.stdout): ncs_group = ncs_object.ncs_groups()[0] all_value_lists = [] for c, t, r in zip(ncs_group.centers(), ncs_group.translations_orth(), ncs_group.rota_matrices()): new_sites_cart = flex.vec3_double() r_inv = r.inverse() for site in sites_orth: new_sites_cart.append(r_inv * (matrix.col(site) - t)) # get value at new_sites cart and make sure they are all the same... new_sites_fract = crystal_symmetry.unit_cell().fractionalize(new_sites_cart) values = flex.double() for site_fract in new_sites_fract: values.append(map_data.value_at_closest_grid_point(site_fract)) all_value_lists.append(values) return get_cc_among_value_lists(all_value_lists) def get_cc_among_value_lists(all_value_lists): a = all_value_lists[0] cc_avg = 0. cc_low = None cc_n = 0. for j in range(1, len(all_value_lists)): b = all_value_lists[j] cc = flex.linear_correlation(a, b).coefficient() cc_avg+= cc cc_n+= 1. if cc_low is None or cc0: import math return cc_low*math.sqrt(len(all_value_lists)), cc_avg else: return None, None def score_ncs_in_map(map_data = None, ncs_object = None, sites_orth = None, identify_ncs_id = None, ncs_in_cell_only = None, allow_score_with_pg = True, crystal_symmetry = None, out = sys.stdout): if not ncs_object or ncs_object.max_operators()<2: return None, None ncs_group = ncs_object.ncs_groups()[0] # don't use point-group symmetry if we have only some of the ops if allow_score_with_pg and ( (not identify_ncs_id) or ncs_group.is_point_group_symmetry()): return score_ncs_in_map_point_group_symmetry( map_data = map_data, ncs_object = ncs_object, sites_orth = sites_orth, crystal_symmetry = crystal_symmetry, out = out) # This version does not assume point-group symmetry: find the NCS # operator that maps each point on to all others the best, then save # that list of values if (not ncs_group) or not ncs_group.n_ncs_oper(): identify_ncs_id_list = [None] else: identify_ncs_id_list = list(range(ncs_group.n_ncs_oper()))+[None] all_value_lists = [] if not sites_orth: sites_orth = get_points_in_map(map_data, n = 100, minimum_fraction_of_max = 0.05, crystal_symmetry = crystal_symmetry) for site in sites_orth: best_id = 0 best_score = None best_values = None for site_ncs_id in identify_ncs_id_list: #last is real one if site_ncs_id is None: site_ncs_id = best_id real_thing = True else: real_thing = False if identify_ncs_id and site_ncs_id: local_site = ncs_group.rota_matrices()[site_ncs_id] * matrix.col(site) + \ ncs_group.translations_orth()[site_ncs_id] else: local_site = site new_sites_cart = flex.vec3_double() for c, t, r in zip(ncs_group.centers(), ncs_group.translations_orth(), ncs_group.rota_matrices()): r_inv = r.inverse() new_sites_cart.append(r_inv * (matrix.col(local_site) - t)) new_sites_fract = crystal_symmetry.unit_cell().fractionalize( new_sites_cart) values = flex.double() for site_frac in new_sites_fract: if (not ncs_in_cell_only) or ( site_frac[0]>= 0 and site_frac[0]<= 1 and \ site_frac[1]>= 0 and site_frac[1]<= 1 and \ site_frac[2]>= 0 and site_frac[2]<= 1): values.append(map_data.value_at_closest_grid_point(site_frac)) else: values.append(0.) score = values.standard_deviation_of_the_sample() if real_thing or (best_score is None or score < best_score): best_score = score best_id = site_ncs_id best_values = values all_value_lists.append(best_values) values_by_site_dict = {} # all_value_lists[j][i] -> values_by_site_dict[i][j] # there are sites_orth.size() values of j # there are len(ncs_group_centers) == len(all_value_lists[0]) values of i for i in range(len(all_value_lists[0])): values_by_site_dict[i] = flex.double() # value_list[0][1] for j in range(sites_orth.size()): values_by_site_dict[i].append(all_value_lists[j][i]) new_all_values_lists = [] for i in range(len(all_value_lists[0])): new_all_values_lists.append(values_by_site_dict[i]) score, cc = get_cc_among_value_lists(new_all_values_lists) return score, cc def get_points_in_map(map_data, n = None, minimum_fraction_of_max = 0., random_xyz = None, max_tries_ratio = 100, crystal_symmetry = None): map_1d = map_data.as_1d() map_mean = map_1d.min_max_mean().mean map_max = map_1d.min_max_mean().max minimum_value = map_mean+minimum_fraction_of_max*(map_max-map_mean) points_list = flex.vec3_double() import random random.seed(1) nu, nv, nw = map_data.all() xyz_fract = crystal_symmetry.unit_cell().fractionalize( tuple((17.4, 27.40128571, 27.32985714, ))) for i in range(int(max_tries_ratio*n)): # max tries ix = random.randint(0, nu-1) iy = random.randint(0, nv-1) iz = random.randint(0, nw-1) xyz_fract = matrix.col((ix/nu, iy/nv, iz/nw, )) value = map_data.value_at_closest_grid_point(xyz_fract) if value > minimum_value and value = n: break sites_orth = crystal_symmetry.unit_cell().orthogonalize(points_list) return sites_orth def get_ncs_list(params = None, symmetry = None, symmetry_center = None, helical_rot_deg = None, helical_trans_z_angstrom = None, op_max = None, two_fold_along_x = None, ncs_obj_to_check = None, crystal_symmetry = None, map_data = None, include_helical_symmetry = None, max_helical_ops_to_check = None, require_helical_or_point_group_symmetry = None, out = sys.stdout): # params.reconstruction_symmetry.require_helical_or_point_group_symmetry # params.reconstruction_symmetry.include_helical_symmetry): # params.reconstruction_symmetry.max_helical_ops_to_check)) if ncs_obj_to_check and ncs_obj_to_check.max_operators()>1: return [ncs_obj_to_check] # , ["SUPPLIED NCS"] from mmtbx.ncs.ncs import generate_ncs_ops ncs_list = generate_ncs_ops( must_be_consistent_with_space_group_number = \ params.reconstruction_symmetry.must_be_consistent_with_space_group_number, symmetry = symmetry, helical_rot_deg = helical_rot_deg, helical_trans_z_angstrom = helical_trans_z_angstrom, op_max = op_max, two_fold_along_x = two_fold_along_x, include_helical_symmetry = \ params.reconstruction_symmetry.include_helical_symmetry, max_helical_ops_to_check = \ params.reconstruction_symmetry.max_helical_ops_to_check, require_helical_or_point_group_symmetry = \ params.reconstruction_symmetry.require_helical_or_point_group_symmetry, out = out) # Generate helical symmetry from map if necessary if symmetry.lower() == 'helical' or ( symmetry.lower() in ['all', 'any'] and params.reconstruction_symmetry.include_helical_symmetry): if helical_rot_deg is None or helical_trans_z_angstrom is None: # returns ncs for symmetry_center at symmetry_center ncs_object, helical_rot_deg, helical_trans_z_angstrom = \ find_helical_symmetry(params = params, symmetry_center = symmetry_center, map_data = map_data, crystal_symmetry = crystal_symmetry, out = out) if ncs_object: ncs_name = "Type: Helical %5.2f deg %6.2f Z-trans " %( helical_rot_deg, helical_trans_z_angstrom) ncs_object.set_ncs_name(ncs_name) ncs_list.append(ncs_object) if symmetry_center and tuple(symmetry_center) != (0, 0, 0, ): print("Offsetting NCS center by (%.2f, %.2f, %.2f) A " %(tuple(symmetry_center)), file = out) new_list = [] for ncs_obj in ncs_list: new_list.append(ncs_obj.coordinate_offset(coordinate_offset = symmetry_center)) ncs_list = new_list if (require_helical_or_point_group_symmetry): for ncs_obj in ncs_list: assert ncs_obj.is_helical_along_z() or ncs_obj.is_point_group_symmetry() return ncs_list def find_helical_symmetry(params = None, symmetry_center = None, map_data = None, crystal_symmetry = None, max_z_to_test = 2, max_peaks_to_score = 5, out = sys.stdout): params = deepcopy(params) # so changing them does not go back if not params.crystal_info.resolution: from cctbx.maptbx import d_min_from_map params.crystal_info.resolution = d_min_from_map( map_data, crystal_symmetry.unit_cell(), resolution_factor = 1./4.) if str(params.reconstruction_symmetry.score_basis) == 'None': params.reconstruction_symmetry.score_basis = 'cc' if params.reconstruction_symmetry.smallest_object is None: params.reconstruction_symmetry.smallest_object = \ 5*params.crystal_info.resolution print("\nLooking for helical symmetry with center at (%.2f, %.2f, %.2f) A\n" %( tuple(symmetry_center)), file = out) print("\nFinding likely translations along z...", file = out) map_coeffs, dummy = get_f_phases_from_map(map_data = map_data, crystal_symmetry = crystal_symmetry, d_min = params.crystal_info.resolution, return_as_map_coeffs = True, out = out) f_array, phases = map_coeffs_as_fp_phi(map_coeffs) from cctbx.maptbx.refine_sharpening import quasi_normalize_structure_factors (d_max, d_min) = f_array.d_max_min() f_array.setup_binner(d_max = d_max, d_min = d_min, n_bins = 20) normalized = quasi_normalize_structure_factors( f_array, set_to_minimum = 0.01) # Now look along c* and get peaks zero_index_a = 0 # look along c* zero_index_b = 1 c_star_list = get_c_star_list(f_array = normalized, zero_index_a = zero_index_a, zero_index_b = zero_index_b) likely_z_translations = get_helical_trans_z_angstrom(params = params, c_star_list = c_star_list, crystal_symmetry = crystal_symmetry, out = out) # Now for each z...get the rotation. Try +/- z and try rotations abc = crystal_symmetry.unit_cell().parameters()[:3] max_z = max(abc[0], abc[1]) if params.reconstruction_symmetry.max_helical_rotations_to_check: min_delta_rot = 360/max(1, params.reconstruction_symmetry.max_helical_rotations_to_check) else: min_delta_rot = 0.01 delta_rot = max(min_delta_rot, (180./3.141659)*params.crystal_info.resolution/max_z) delta_z = params.crystal_info.resolution/4. print("\nOptimizing helical paramers:", file = out) print("\n Rot Trans Score CC", file = out) n_rotations = int(0.5+360/delta_rot) rotations = [] for k in range(1, n_rotations): helical_rot_deg = k*delta_rot if helical_rot_deg > 180: helical_rot_deg = helical_rot_deg-360 rotations.append(helical_rot_deg) done = False n_try = 0 score_list = [] quick = params.control.quick best_ncs_cc = None best_ncs_obj = None best_score = None best_helical_trans_z_angstrom = None best_helical_rot_deg = None import math for helical_trans_z_angstrom in likely_z_translations[:max_z_to_test]: n_try+= 1 if done: break for helical_rot_deg in rotations: if done: break if abs(helical_trans_z_angstrom)+abs(math.sin( helical_rot_deg*(3.14159/180.))*max_z) < \ params.reconstruction_symmetry.smallest_object: continue # this is identity new_ncs_obj, ncs_cc, ncs_score, \ new_helical_trans_z_angstrom, new_helical_rot_deg = try_helical_params( optimize = 0, helical_rot_deg = helical_rot_deg, helical_trans_z_angstrom = helical_trans_z_angstrom, params = params, map_data = map_data, map_symmetry_center = symmetry_center, # should not be needed XXX symmetry_center = symmetry_center, crystal_symmetry = crystal_symmetry, out = null_out()) if not ncs_score: continue if ncs_cc> 0.95 or \ ncs_cc > 2*params.reconstruction_symmetry.min_ncs_cc or \ (quick and (ncs_cc> 0.90 or ncs_cc > 1.5*params.reconstruction_symmetry.min_ncs_cc)): print(" %.2f %.2f %.2f %.2f (ok to go on)" %( new_helical_rot_deg, new_helical_trans_z_angstrom, ncs_score, ncs_cc), file = out) done = True if params.control.verbose: print(" %.2f %.2f %.2f %.2f" %( new_helical_rot_deg, new_helical_trans_z_angstrom, ncs_score, ncs_cc), file = out) score_list.append( [ncs_score, ncs_cc, new_helical_rot_deg, new_helical_trans_z_angstrom]) score_list.sort(key=itemgetter(0)) score_list.reverse() done = False for ncs_score, ncs_cc, helical_rot_deg, helical_trans_z_angstrom in \ score_list[:max_peaks_to_score]: if done: continue # rescore and optimize: new_ncs_obj, ncs_cc, ncs_score, \ new_helical_trans_z_angstrom, new_helical_rot_deg = try_helical_params( params = params, best_score = best_score, helical_rot_deg = helical_rot_deg, helical_trans_z_angstrom = helical_trans_z_angstrom, delta_z = delta_z/2., delta_rot = delta_rot/2., map_data = map_data, map_symmetry_center = symmetry_center, symmetry_center = symmetry_center, crystal_symmetry = crystal_symmetry, out = null_out()) if not ncs_score or ncs_score <0: continue if best_score is None or ncs_score>best_score: best_ncs_cc = ncs_cc best_ncs_obj = new_ncs_obj best_score = ncs_score best_helical_trans_z_angstrom = new_helical_trans_z_angstrom best_helical_rot_deg = new_helical_rot_deg # after trying out a range quit if good enough if best_ncs_cc> 0.90 or \ best_ncs_cc > 1.5*params.reconstruction_symmetry.min_ncs_cc or \ ( (quick or n_try>1) and \ ncs_cc>= params.reconstruction_symmetry.min_ncs_cc): print(" %.2f %.2f %.2f %.2f (high enough to go on)" %( best_helical_rot_deg, best_helical_trans_z_angstrom, best_score, best_ncs_cc), file = out) done = True # Optimize one more time trying fractional values, but only if # that makes the delta less than the resolution print("\nTrying fraction of rot/trans", file = out) for iter in [0, 1]: if not best_helical_rot_deg: continue for ifract in range(2, 11): if iter == 0: # try fractional test_helical_rot_deg = best_helical_rot_deg/ifract test_helical_trans_z_angstrom = best_helical_trans_z_angstrom/ifract if test_helical_trans_z_angstrom > params.crystal_info.resolution*1.1: continue # skip it...would have been a peak if ok else: # iter >0 if ifract > 0: continue # skip these else: # optimize current best test_helical_rot_deg = best_helical_rot_deg test_helical_trans_z_angstrom = best_helical_trans_z_angstrom new_ncs_obj, new_ncs_cc, new_ncs_score, \ new_helical_trans_z_angstrom, new_helical_rot_deg = \ try_helical_params( params = params, helical_rot_deg = test_helical_rot_deg, helical_trans_z_angstrom = test_helical_trans_z_angstrom, delta_z = delta_z/2., delta_rot = delta_rot/2., map_data = map_data, map_symmetry_center = symmetry_center, symmetry_center = symmetry_center, crystal_symmetry = crystal_symmetry, out = out) if new_ncs_score is not None: new_ncs_score = new_ncs_score*\ params.reconstruction_symmetry.scale_weight_fractional_translation # give slight weight to smaller if best_score is None or new_ncs_score > best_score: best_ncs_cc = new_ncs_cc best_ncs_obj = new_ncs_obj best_score = new_ncs_score best_helical_trans_z_angstrom = new_helical_trans_z_angstrom best_helical_rot_deg = new_helical_rot_deg print(" %.2f %.2f %.2f %.2f (improved fractions)" %( best_helical_rot_deg, best_helical_trans_z_angstrom, best_score, best_ncs_cc), file = out) else: print(" %.2f %.2f %.2f %.2f (worse with fractions)" %( new_helical_rot_deg, new_helical_trans_z_angstrom, new_ncs_score, new_ncs_cc), file = out) # Optimize one more time trying multiples of values to get better param imult = int(0.5+ 0.33*max_z/params.reconstruction_symmetry.max_helical_ops_to_check) working_ncs_cc = best_ncs_cc working_ncs_obj = best_ncs_obj working_score = None if best_helical_rot_deg: working_helical_rot_deg = best_helical_rot_deg*imult working_helical_trans_z_angstrom = best_helical_trans_z_angstrom*imult else: working_helical_rot_deg = None working_helical_trans_z_angstrom = None imult = 0 if imult > 1: print("\nTrying %sx multiples of rot/trans" %(imult), file = out) improved = False for iter in [1, 2, 3]: if iter > 1 and not improved: break improved = False new_ncs_obj, new_ncs_cc, new_ncs_score, \ new_helical_trans_z_angstrom, new_helical_rot_deg = \ try_helical_params( params = params, helical_rot_deg = working_helical_rot_deg, helical_trans_z_angstrom = working_helical_trans_z_angstrom, delta_z = delta_z/(2.*iter), delta_rot = delta_rot/(2.*iter), map_data = map_data, map_symmetry_center = symmetry_center, symmetry_center = symmetry_center, crystal_symmetry = crystal_symmetry, out = out) if new_ncs_score is not None: if working_score is None or new_ncs_score > working_score: working_ncs_cc = new_ncs_cc working_ncs_obj = new_ncs_obj working_score = new_ncs_score working_helical_trans_z_angstrom = new_helical_trans_z_angstrom working_helical_rot_deg = new_helical_rot_deg print(" %.2f %.2f %.2f %.2f (Scoring for multiple)" %( working_helical_rot_deg, working_helical_trans_z_angstrom, working_score, working_ncs_cc), file = out) # and rescore with this: working_helical_rot_deg = working_helical_rot_deg/imult working_helical_trans_z_angstrom = working_helical_trans_z_angstrom/imult for iter in [1, 2, 3]: new_ncs_obj, new_ncs_cc, new_ncs_score, \ new_helical_trans_z_angstrom, new_helical_rot_deg = \ try_helical_params( params = params, helical_rot_deg = working_helical_rot_deg, helical_trans_z_angstrom = working_helical_trans_z_angstrom, delta_z = delta_z/(2.*iter), delta_rot = delta_rot/(2.*iter), map_data = map_data, map_symmetry_center = symmetry_center, symmetry_center = symmetry_center, crystal_symmetry = crystal_symmetry, out = null_out()) if new_ncs_score is not None: working_ncs_obj, working_ncs_cc, working_ncs_score, \ working_helical_trans_z_angstrom, working_helical_rot_deg = \ new_ncs_obj, new_ncs_cc, new_ncs_score, \ new_helical_trans_z_angstrom, new_helical_rot_deg if best_score is None or working_ncs_score > best_score: if params.control.verbose: print("\nTaking parameters from multiples", file = out) best_ncs_cc = working_ncs_cc best_ncs_obj = working_ncs_obj best_score = working_ncs_score best_helical_trans_z_angstrom = working_helical_trans_z_angstrom best_helical_rot_deg = working_helical_rot_deg print(" %.2f %.2f %.2f %.2f (improved by multiples)" %( best_helical_rot_deg, best_helical_trans_z_angstrom, best_score, best_ncs_cc), file = out) improved = True working_ncs_cc = best_ncs_cc working_ncs_obj = best_ncs_obj working_score = None working_helical_trans_z_angstrom = best_helical_trans_z_angstrom*imult working_helical_rot_deg = best_helical_rot_deg*imult if best_helical_rot_deg and best_helical_trans_z_angstrom and best_score and best_ncs_cc: # Check to make sure there is no overlap print(" %.2f %.2f %.2f %.2f (Final)" %( best_helical_rot_deg, best_helical_trans_z_angstrom, \ best_score, best_ncs_cc), file = out) from mmtbx.ncs.ncs import get_helical_symmetry ncs_object = get_helical_symmetry( helical_rot_deg = best_helical_rot_deg, helical_trans_z_angstrom = best_helical_trans_z_angstrom, max_ops = params.reconstruction_symmetry.max_helical_ops_to_check) else: ncs_object = None return ncs_object, best_helical_rot_deg, best_helical_trans_z_angstrom def try_helical_params( optimize = None, best_score = None, delta_z = None, delta_rot = None, helical_rot_deg = None, helical_trans_z_angstrom = None, params = None, map_data = None, map_symmetry_center = None, symmetry_center = None, crystal_symmetry = None, out = sys.stdout): if delta_z is None or delta_rot is None: assert not optimize local_params = deepcopy(params) local_params.reconstruction_symmetry.min_ncs_cc = -100 local_params.reconstruction_symmetry.n_rescore = 0 local_params.reconstruction_symmetry.symmetry = 'helical' local_params.reconstruction_symmetry.helical_rot_deg = helical_rot_deg local_params.reconstruction_symmetry.helical_trans_z_angstrom = \ helical_trans_z_angstrom abc = crystal_symmetry.unit_cell().parameters()[:3] max_z = max(abc[0], abc[1]) import math if abs(helical_trans_z_angstrom)+abs(math.sin( helical_rot_deg*(3.14159/180.))*max_z) < \ params.reconstruction_symmetry.smallest_object: return None, None, None, \ None, None best_helical_trans_z_angstrom, best_helical_rot_deg = \ helical_trans_z_angstrom, helical_rot_deg best_ncs_score = best_score best_ncs_obj = None best_ncs_cc = None test_ncs_obj, test_ncs_cc, test_ncs_score = get_ncs_from_map(params = local_params, map_data = map_data, map_symmetry_center = symmetry_center, symmetry_center = symmetry_center, crystal_symmetry = crystal_symmetry, out = null_out()) if params.reconstruction_symmetry.score_basis == 'cc': test_ncs_score = test_ncs_cc if best_ncs_score is None or test_ncs_score>best_ncs_score: best_ncs_score = test_ncs_score best_ncs_cc = test_ncs_cc best_ncs_obj = test_ncs_obj if optimize is None: optimize = params.reconstruction_symmetry.max_helical_optimizations # save in case we need to go back working_helical_trans_z_angstrom, working_helical_rot_deg = \ helical_trans_z_angstrom, helical_rot_deg working_ncs_score = best_ncs_score working_ncs_cc = best_ncs_cc working_ncs_obj = best_ncs_obj if optimize and (best_score is None or best_ncs_score > best_score): # try with few to many operators.. if params.control.verbose: print("\nOptimizing by varying number of operators", file = out) print("Starting score: %.2f" %(working_ncs_score), file = out) for k in range(optimize): local_params.reconstruction_symmetry.max_helical_ops_to_check = min(k+1, params.reconstruction_symmetry.max_helical_ops_to_check) best_score = None # start over for each number of operators for i in [0, -1, 1]: for j in [0, -1, 1]: new_ncs_obj, new_ncs_cc, new_ncs_score, \ new_helical_trans_z_angstrom, new_helical_rot_deg = try_helical_params( optimize = False, helical_rot_deg = max(0.1, best_helical_rot_deg+i*delta_rot), helical_trans_z_angstrom = max(0.01, best_helical_trans_z_angstrom+j*delta_z), delta_z = delta_z, delta_rot = delta_rot, params = params, map_data = map_data, map_symmetry_center = symmetry_center, symmetry_center = symmetry_center, crystal_symmetry = crystal_symmetry, out = out) if new_ncs_score and new_ncs_score>0 and ( best_score is None or new_ncs_score>best_score): if params.control.verbose: print("Working score for %s ops: %.2f" %( local_params.reconstruction_symmetry.max_helical_ops_to_check, new_ncs_score), file = out) best_score = new_ncs_score best_ncs_score = new_ncs_score best_helical_trans_z_angstrom = new_helical_trans_z_angstrom best_helical_rot_deg = new_helical_rot_deg best_ncs_obj = new_ncs_obj best_ncs_cc = new_ncs_cc delta_rot = delta_rot/2 delta_z = delta_z/2 #rescore with what we now have (best values) and compare with working local_params.reconstruction_symmetry.max_helical_ops_to_check = \ params.reconstruction_symmetry.max_helical_ops_to_check if params.control.verbose: print("Rescoring with original number of operators (%s)" %( local_params.reconstruction_symmetry.max_helical_ops_to_check), file = out) local_params.reconstruction_symmetry.helical_rot_deg = best_helical_rot_deg local_params.reconstruction_symmetry.helical_trans_z_angstrom = \ best_helical_trans_z_angstrom best_ncs_obj, best_ncs_cc, best_ncs_score = get_ncs_from_map( params = local_params, map_data = map_data, map_symmetry_center = symmetry_center, symmetry_center = symmetry_center, crystal_symmetry = crystal_symmetry, out = null_out()) if params.reconstruction_symmetry.score_basis == 'cc': best_ncs_score = best_ncs_cc # now take it if better if best_ncs_cc and best_ncs_cc>working_ncs_cc: if params.control.verbose: print("Using optimized values (%.2f > %.2f)" %( best_ncs_cc, working_ncs_cc), file = out) # keep these (best) else: if params.control.verbose: print("Rejecting optimized values (%.2f <= %.2f)" %( best_ncs_cc, working_ncs_cc), file = out) # resture working values best_helical_trans_z_angstrom, best_helical_rot_deg = \ working_helical_trans_z_angstrom, working_helical_rot_deg best_ncs_score = working_ncs_score best_ncs_obj = working_ncs_obj best_ncs_cc = working_ncs_cc if params.reconstruction_symmetry.score_basis == 'cc': best_ncs_score = best_ncs_cc return best_ncs_obj, best_ncs_cc, best_ncs_score, \ best_helical_trans_z_angstrom, best_helical_rot_deg def get_d_and_value_list(c_star_list): d_list = [] from scitbx.array_family import flex value_list = flex.double() for hkl, value, d in c_star_list: d_list.append(d) value_list.append(value) max_value = value_list.min_max_mean().max if value_list.size()>3: max_value = value_list[2] new_d_list = [] new_value_list = [] for d, value in zip(d_list, value_list): if value < max_value/1000: # reject those that are really zero continue new_d_list.append(d) new_value_list.append(value) return new_d_list, new_value_list def get_helical_trans_z_angstrom(params = None, c_star_list = None, crystal_symmetry = None, minimum_ratio = 2., max_first_peak = 1, out = sys.stdout): # Find highest-resolution peak along c*...guess it is n*z_translation # where n is small max_z = flex.double(crystal_symmetry.unit_cell().parameters()[:3]).min_max_mean().max if params.control.verbose: print("Values along c*: ", file = out) d_list, value_list = get_d_and_value_list(c_star_list) for d, value in zip(d_list, value_list): if params.control.verbose: print(" %.2f A : %.2f " %(d, value), file = out) d_list, value_list = get_max_min_list( d_list = d_list, value_list = value_list, minimum_ratio = 1.0) d_list, value_list = get_max_min_list( d_list = d_list, value_list = value_list, minimum_ratio = 2.0, maximum_only = True) sort_list = [] for d, value in zip(d_list, value_list): sort_list.append([value, d]) sort_list.sort(key=itemgetter(0)) sort_list.reverse() likely_z_translations = [] dis_min = params.crystal_info.resolution/4 for value, d in sort_list: likely_z_translations_local = [] for i in range(1, max_first_peak+1): z = d/i if z > max_z: continue if z < dis_min: continue # no way if is_close_to_any(z = z, others = likely_z_translations, dis_min = dis_min): continue likely_z_translations_local.append(z) likely_z_translations.append(z) print("\nMaximal values along c* and likely z-translations: ", file = out) for z in likely_z_translations: print(" %.2f A " %(z), end = ' ', file = out) print(file = out) return likely_z_translations def small_deltas(z_translations, dis_min = None): delta_list = [] for z, z1 in zip(z_translations, z_translations[1:]): delta = abs(z-z1) if not is_close_to_any(z = delta, others = z_translations+delta_list, dis_min = dis_min): delta_list.append(abs(delta)) return delta_list def is_close_to_any(z = None, others = None, dis_min = None): for x in others: if abs(x-z)= value_prev *minimum_ratio) and ( value >= value_next*minimum_ratio): # local max max_min_list.append(value) max_min_d_list.append(d) if (not maximum_only) and d and ( value <= value_prev ) and ( value <= value_next): # local min max_min_list.append(value) max_min_d_list.append(d) return max_min_d_list, max_min_list def get_c_star_list(f_array = None, zero_index_a = 0, zero_index_b = 1, zero_index_c = 2): c_star_list = [] for value, (indices, d) in zip(f_array.data(), f_array.d_spacings()): if indices[zero_index_a] == 0 and indices[zero_index_b] == 0 and \ indices[zero_index_c] >= 4: c_star_list.append([tuple(indices), value, d]) c_star_list.sort() return c_star_list def get_params_from_args(args): command_line = iotbx.phil.process_command_line_with_files( map_file_def = "input_files.map_file", seq_file_def = "input_files.seq_file", pdb_file_def = "input_files.pdb_in", ncs_file_def = "input_files.ncs_file", args = args, master_phil = master_phil) return command_line.work.extract() def get_mask_around_molecule(map_data = None, wang_radius = None, buffer_radius = None, force_buffer_radius = None, return_masked_fraction = False, minimum_fraction_of_max = 0.01, solvent_content = None, solvent_content_iterations = None, crystal_symmetry = None, out = sys.stdout): # use iterated solvent fraction tool to identify mask around molecule try: from phenix.autosol.map_to_model import iterated_solvent_fraction solvent_fraction, mask = iterated_solvent_fraction( crystal_symmetry = crystal_symmetry, wang_radius = wang_radius, solvent_content = solvent_content, solvent_content_iterations = solvent_content_iterations, map_as_double = map_data, out = out) except Exception as e: print("No mask obtained...", file = out) return None, None if not mask: print("No mask obtained...", file = out) return None, None # Now expand the mask to increase molecular region expand_size = estimate_expand_size( crystal_symmetry = crystal_symmetry, map_data = map_data, expand_target = buffer_radius, minimum_expand_size = 0, out = out) print("Target mask expand size is %d based on buffer_radius of %7.1f A" %( expand_size, buffer_radius), file = out) co, sorted_by_volume, min_b, max_b = get_co(map_data = mask, threshold = 0.5, wrapping = False) if len(sorted_by_volume)<2: print ("\nSkipping expansion as no space is available\n", file = out) return None, None masked_fraction = sorted_by_volume[1][0]/mask.size() if expand_size <= 0: expanded_fraction = masked_fraction else: # Try to get expanded fraction < 0.5*(masked_fraction+1) upper_limit = 0.5*(masked_fraction+1) bool_region_mask = co.expand_mask(id_to_expand = sorted_by_volume[1][1], expand_size = expand_size) s = (bool_region_mask == True) expanded_fraction = s.count(True)/s.size() if expanded_fraction>upper_limit: amount_too_big = max(1.e-10, expanded_fraction-upper_limit)/max(1.e-10, expanded_fraction-masked_fraction)**0.667 # cut back original_expand_size = expand_size expand_size = max(1, int(0.5+expand_size * min(1, max(0, (1-amount_too_big))))) if expand_size != original_expand_size: print ("\nCutting back expand size to try and get "+ "fraction < about %.2f . New expand_size: %s" %( upper_limit, expand_size), file = out) bool_region_mask = co.expand_mask(id_to_expand = sorted_by_volume[1][1], expand_size = expand_size) s = (bool_region_mask == True) expanded_fraction = s.count(True)/s.size() if expanded_fraction > 0.9999: print ("\nSkipping expansion as no space is available\n", file = out) return None, None print("\nLargest masked region before buffering: %7.2f" %(masked_fraction), file = out) print("\nLargest masked region after buffering: %7.2f" %(expanded_fraction), file = out) if solvent_content and (not force_buffer_radius): delta_as_is = abs(solvent_content- (1-masked_fraction)) delta_expanded = abs(solvent_content- (1-expanded_fraction)) if delta_expanded > delta_as_is: # already there expand_size = 0 print ("Setting expand size to zero as masked fraction already ", "close to solvent_content", file = out) s = None minimum_size = sorted_by_volume[1][0] * minimum_fraction_of_max if expand_size == 0: result = co.result() else: result = None for v1, i1 in sorted_by_volume[1:]: if v1 < minimum_size: break if expand_size > 0: bool_region_mask = co.expand_mask( id_to_expand = i1, expand_size = expand_size) else: bool_region_mask = (result == i1) if s is None: s = (bool_region_mask == True) else: s |= (bool_region_mask == True) mask.set_selected(s, 1) mask.set_selected(~s, 0) masked_fraction = mask.count(1)/mask.size() print("Masked fraction after buffering: %7.2f" %(masked_fraction), file = out) if return_masked_fraction: return mask.as_double(), 1-masked_fraction else: # usual return solvent fraction estimate return mask.as_double(), solvent_fraction # This is solvent fraction est. def get_mean_in_and_out(sel = None, map_data = None, verbose = False, out = sys.stdout): mean_value_in, fraction_in = get_mean_in_or_out(sel = sel, map_data = map_data, out = out) mean_value_out, fraction_out = get_mean_in_or_out(sel = ~sel, map_data = map_data, out = out) if mean_value_out is None: mean_value_out = mean_value_in if mean_value_in is None: mean_value_in = mean_value_out if verbose: print("\nMean inside mask: %7.2f Outside mask: %7.2f Fraction in: %7.2f" %( mean_value_in, mean_value_out, fraction_in), file = out) return mean_value_in, mean_value_out, fraction_in def get_mean_in_or_out(sel = None, map_data = None, out = sys.stdout): masked_map = map_data.deep_copy() if sel.count(False) != 0: masked_map.as_1d().set_selected(~sel.as_1d(), 0) mean_after_zeroing_in_or_out = masked_map.as_1d().min_max_mean().mean if sel.count(True) != 0: masked_map.as_1d().set_selected(sel.as_1d(), 1) fraction_in_or_out = masked_map.as_1d().min_max_mean().mean if fraction_in_or_out >1.e-10: mean_value = mean_after_zeroing_in_or_out/fraction_in_or_out else: mean_value = None return mean_value, fraction_in_or_out def apply_soft_mask(map_data = None, mask_data = None, rad_smooth = None, crystal_symmetry = None, set_outside_to_mean_inside = False, set_mean_to_zero = False, threshold = 0.5, verbose = False, out = sys.stdout): # apply a soft mask based on mask_data to map_data. # set value outside mask == mean value inside mask or mean value outside mask original_mask_data = mask_data.deep_copy() smoothed_mask_data = smooth_mask_data(mask_data = mask_data, crystal_symmetry = crystal_symmetry, threshold = threshold, rad_smooth = rad_smooth) masked_map = apply_mask_to_map(mask_data = original_mask_data, smoothed_mask_data = smoothed_mask_data, map_data = map_data, set_outside_to_mean_inside = set_outside_to_mean_inside, set_mean_to_zero = set_mean_to_zero, threshold = threshold, verbose = verbose, out = out) return masked_map, smoothed_mask_data def smooth_mask_data(mask_data = None, crystal_symmetry = None, threshold = None, rad_smooth = None): if threshold is None: threshold = mask_data.as_1d().min_max_mean().max * 0.5 # Smooth a mask in place. First make it a binary mask s = mask_data > threshold # s marks inside mask mask_data = mask_data.set_selected(~s, 0) # outside mask == 0 mask_data = mask_data.set_selected( s, 1) if mask_data.count(1) and mask_data.count(0): # something to do maptbx.unpad_in_place(map = mask_data) mask_data = maptbx.smooth_map( map = mask_data, crystal_symmetry = crystal_symmetry, rad_smooth = rad_smooth) # Make sure that mask_data max value is now 1, scale if not max_mask_data_value = mask_data.as_1d().min_max_mean().max if max_mask_data_value > 1.e-30 and max_mask_data_value!= 1.0: mask_data = mask_data*(1./max_mask_data_value) else: pass return mask_data def apply_mask_to_map(mask_data = None, smoothed_mask_data = None, set_outside_to_mean_inside = None, set_mean_to_zero = None, map_data = None, threshold = 0.5, verbose = None, out = sys.stdout): if smoothed_mask_data and not mask_data: mask_data = smoothed_mask_data elif mask_data and not smoothed_mask_data: smoothed_mask_data = mask_data s = mask_data > threshold # s marks inside mask # get mean inside or outside mask if verbose: print("\nStarting map values inside and outside mask:", file = out) mean_value_in, mean_value_out, fraction_in = get_mean_in_and_out(sel = s, verbose = verbose, map_data = map_data, out = out) if verbose: print("\nMask inside and outside values", file = out) mask_mean_value_in, mask_mean_value_out, mask_fraction_in = get_mean_in_and_out( sel = s, map_data = mask_data, verbose = verbose, out = out) if verbose: print("\nSmoothed mask inside and outside values", file = out) smoothed_mean_value_in, smoothed_mean_value_out, smoothed_fraction_in = \ get_mean_in_and_out(sel = s, map_data = smoothed_mask_data, verbose = verbose, out = out) # Now replace value outside mask with mean_value, value inside with current, # smoothly going from one to the other based on mask_data # set_to_mean will be a constant map with value equal to inside or outside if (set_outside_to_mean_inside is False): target_value_for_outside = 0 elif set_outside_to_mean_inside or mean_value_out is None: target_value_for_outside = mean_value_in if verbose: print("Setting value outside mask to mean inside (%.2f)" %( target_value_for_outside), file = out) else: target_value_for_outside = mean_value_out if verbose: print("Setting value outside mask to mean outside (%.2f)" %( target_value_for_outside), file = out) set_to_mean = mask_data.deep_copy() ss = set_to_mean > -1.e+30 # select everything set_to_mean.set_selected(ss, target_value_for_outside) masked_map = (map_data * smoothed_mask_data ) + (set_to_mean * (1-smoothed_mask_data)) if set_mean_to_zero: # remove average masked_map = masked_map - masked_map.as_1d().min_max_mean().mean if verbose: print("\nFinal mean value inside and outside mask:", file = out) mean_value_in, mean_value_out, fraction_in = get_mean_in_and_out(sel = s, map_data = masked_map, verbose = verbose, out = out) return masked_map def estimate_expand_size( crystal_symmetry = None, map_data = None, expand_target = None, minimum_expand_size = 1, out = sys.stdout): if not expand_target: return minimum_expand_size abc = crystal_symmetry.unit_cell().parameters()[:3] N_ = map_data.all() nn = 0. for i in range(3): delta = abc[i]/N_[i] nn+= expand_target/delta nn = max(1, int(0.5+nn/3.)) print("Expand size (grid units): %d (about %4.1f A) " %( nn, nn*abc[0]/N_[0]), file = out) return max(minimum_expand_size, nn) def get_max_z_range_for_helical_symmetry(params, out = sys.stdout): if not params.input_files.ncs_file: return ncs_obj, dummy_tracking_data = get_ncs(params = params, out = out) if not ncs_obj.is_helical_along_z(): return if params.map_modification.restrict_z_distance_for_helical_symmetry: #take it return params.map_modification.restrict_z_distance_for_helical_symmetry if not params.map_modification.restrict_z_turns_for_helical_symmetry: return print("Identifying maximum z-range for helical symmetry", file = out) print("Maximum of %7.1f turns up and down in Z allowed..." %( params.map_modification.restrict_z_turns_for_helical_symmetry), file = out) r, t = ncs_obj.ncs_groups()[0].helix_rt_forwards() cost = r[0] sint = r[1] import math theta = abs(180.*math.atan2(sint, cost)/3.14159) trans = abs(t) pitch = trans*360./max(0.1, theta) max_z = params.map_modification.restrict_z_turns_for_helical_symmetry*pitch print("Z-values restricted to +/- %7.1f A" %(max_z), file = out) print("\nRunning map-box once to get position of molecule, again to"+\ " apply\n Z restriction\n", file = out) return max_z def dist(x, y): dd = 0. for a, b in zip(x, y): dd+= (a-b)**2 return dd**0.5 def get_ncs_closest_sites( closest_sites = None, sites_cart = None, used_ncs_id_list = None, box_ncs_object = None, box_crystal_symmetry = None, out = sys.stdout): # try to find NCS ops mapping sites_cart close to closest_sites best_id = None best_rms = None best_sites = closest_sites.deep_copy() for ncs_id in range(box_ncs_object.max_operators()): if ncs_id in used_ncs_id_list: continue test_sites = closest_sites.deep_copy() ncs_sites_cart = get_ncs_sites_cart(sites_cart = sites_cart, ncs_obj = box_ncs_object, unit_cell = box_crystal_symmetry.unit_cell(), ncs_id = ncs_id, ncs_in_cell_only = False) test_sites.extend(ncs_sites_cart) rms = radius_of_gyration_of_vector(test_sites) if best_rms is None or rms < best_rms: best_rms = rms best_ncs_id = ncs_id best_sites = test_sites.deep_copy() used_ncs_id_list.append(best_ncs_id) return best_sites, used_ncs_id_list def get_closest_sites( high_points = None, sites_cart = None, box_ncs_object = None, box_crystal_symmetry = None, out = sys.stdout): if not box_ncs_object.is_point_group_symmetry() and not \ box_ncs_object.is_helical_along_z(): # extract point_group symmetry if present and box_ncs_object doesn't have it print("Trying to extract point-group symmetry from box_ncs_object "+\ "with %d ops" %( box_ncs_object.max_operators()), file = out) ncs_object = box_ncs_object.deep_copy(extract_point_group_symmetry = True) if ncs_object: print("New number of operators satisfying point-group symmetry: %d" %( ncs_object.max_operators()), file = out) box_ncs_object = ncs_object else: print("No point-group symmetry found", file = out) ncs_copies = box_ncs_object.max_operators() closest_sites = high_points from scitbx.matrix import col for id in range(sites_cart.size()): local_sites_cart = sites_cart[id:id+1] local_sites_cart.extend(get_ncs_sites_cart(sites_cart = local_sites_cart, ncs_obj = box_ncs_object, unit_cell = box_crystal_symmetry.unit_cell(), ncs_in_cell_only = True)) if local_sites_cart.size() = a-boundary_tolerance: x_min = a if y_min>= b-boundary_tolerance: y_min = b if z_min>= c-boundary_tolerance: z_min = c print("\nAdjusted range for box:", file = out) print(" X Y Z", file = out) print(" LOW: %7.1f %7.1f %7.1f " %(tuple([x_min, y_min, z_min])), file = out) print(" HIGH: %7.1f %7.1f %7.1f \n" %(tuple([x_max, y_max, z_max])), file = out) nx, ny, nz = map_data.all() # convert to grid units i_min = max(0, min(nx, int(0.5+nx*x_min/a))) j_min = max(0, min(ny, int(0.5+ny*y_min/b))) k_min = max(0, min(nz, int(0.5+nz*z_min/c))) i_max = max(0, min(nx, int(0.5+nx*x_max/a))) j_max = max(0, min(ny, int(0.5+ny*y_max/b))) k_max = max(0, min(nz, int(0.5+nz*z_max/c))) lower_bounds = [i_min, j_min, k_min] upper_bounds = [i_max, j_max, k_max] print("\nGrid bounds for box:", file = out) print(" X Y Z", file = out) print(" LOW: %7d %7d %7d " %(tuple([i_min, j_min, k_min])), file = out) print(" HIGH: %7d %7d %7d \n" %(tuple([i_max, j_max, k_max])), file = out) return lower_bounds, upper_bounds def get_bounds_for_au_box(params, box = None, out = sys.stdout): # Try to get bounds for a box that include one au if not box.ncs_object or box.ncs_object.max_operators()<2: return None, None, None box_ncs_object = box.ncs_object box_map_data = box.map_box box_crystal_symmetry = box.box_crystal_symmetry random_points = 10*params.reconstruction_symmetry.random_points sites_cart = get_points_in_map(box_map_data, n = random_points, minimum_fraction_of_max = params.segmentation.density_select_threshold, random_xyz = params.crystal_info.resolution*2., crystal_symmetry = box_crystal_symmetry) assert sites_cart.size() >0 # apply symmetry to sites_orth and see where the molecule is ncs_sites_cart = get_ncs_sites_cart(sites_cart = sites_cart, ncs_obj = box_ncs_object, unit_cell = box_crystal_symmetry.unit_cell(), ncs_in_cell_only = True) # generate this in a lower-memory way XXX low_res_map_data = get_low_res_map_data(sites_cart = ncs_sites_cart, d_min = params.crystal_info.resolution*7., crystal_symmetry = box_crystal_symmetry, out = out) high_points = get_high_points_from_map( # actually returns just one. map_data = low_res_map_data, unit_cell = box_crystal_symmetry.unit_cell(), out = out) from scitbx.matrix import col high_points = high_points[0:1] cutout_center = col(high_points[0]) print("Center of box will be near (%7.1f, %7.1f, %7.1f)" %( tuple(cutout_center))) # now figure out box that contains at least one copy of each ncs-related # point. # Find closest ncs-related points for each unique random point to this # center. Starting box is the box that contains all of these. closest_sites = get_closest_sites( high_points = high_points, sites_cart = sites_cart, box_ncs_object = box_ncs_object, box_crystal_symmetry = box_crystal_symmetry, out = out) if not closest_sites or closest_sites.size()<1: print("\nNo sites representing au of map found...skipping au box\n", file = out) return None, None, None print("\nTotal of %d sites representing 1 au found" %( closest_sites.size()), file = out) # write out closest_sites to match original position coordinate_offset = -1*matrix.col(box.shift_cart) write_atoms(file_name = 'one_au.pdb', crystal_symmetry = box_crystal_symmetry, sites = closest_sites+coordinate_offset) unique_closest_sites = closest_sites.deep_copy() # Now if desired, find NCS-related groups of sites if params.segmentation.n_au_box >1: print("\nFinding up to %d related au" %(params.segmentation.n_au_box), file = out) print("Starting RMSD of sites: %7.1f A " %( radius_of_gyration_of_vector(closest_sites)), file = out) sites_orig = closest_sites.deep_copy() used_ncs_id_list = [box_ncs_object.ncs_groups()[0].identity_op_id()] for i in range(params.segmentation.n_au_box-1): closest_sites, used_ncs_id_list = get_ncs_closest_sites( used_ncs_id_list = used_ncs_id_list, closest_sites = closest_sites, sites_cart = sites_orig, box_ncs_object = box_ncs_object, box_crystal_symmetry = box_crystal_symmetry, out = out) print("\nNew total of %d sites representing %d au found" %( closest_sites.size(), params.segmentation.n_au_box), file = out) print("New rmsd: %7.1f A " %( radius_of_gyration_of_vector(closest_sites)), file = out) lower_bounds, upper_bounds = get_range( sites = closest_sites, map_data = box_map_data, boundary_tolerance = params.crystal_info.resolution, unit_cell = box_crystal_symmetry.unit_cell(), out = out) return lower_bounds, upper_bounds, unique_closest_sites+coordinate_offset def get_low_res_map_data(sites_cart = None, crystal_symmetry = None, d_min = None, out = sys.stdout): from cctbx import xray xrs, scatterers = set_up_xrs(crystal_symmetry = crystal_symmetry) unit_cell = crystal_symmetry.unit_cell() sites_fract = unit_cell.fractionalize(sites_cart) for xyz_fract in sites_fract: scatterers.append( xray.scatterer(scattering_type = "H", label = "H", site = xyz_fract, u = 0, occupancy = 1.0)) xrs = xray.structure(xrs, scatterers = scatterers) # generate f_array to d_min with xrs f_array = xrs.structure_factors(d_min = d_min, anomalous_flag = False).f_calc() weight_f_array = f_array.structure_factors_from_scatterers( algorithm = 'direct', xray_structure = xrs).f_calc() return get_map_from_map_coeffs(map_coeffs = weight_f_array, crystal_symmetry = crystal_symmetry) def get_bounds_for_helical_symmetry(params, box = None, crystal_symmetry = None): original_cell = box.map_data.all() new_cell = box.map_box.all() z_first = box.gridding_first[2] z_last = box.gridding_last[2] assert z_last>= z_first z_middle = (z_first+z_last)//2 delta_z = crystal_symmetry.unit_cell().parameters()[5]/box.map_data.all()[2] n_z_max = int(0.5+ params.map_modification.restrict_z_distance_for_helical_symmetry/delta_z) new_z_first = max(z_first, z_middle-n_z_max) new_z_last = min(z_last, z_middle+n_z_max) lower_bounds = list(deepcopy(box.gridding_first)) upper_bounds = list(deepcopy(box.gridding_last)) lower_bounds[2] = new_z_first upper_bounds[2] = new_z_last return lower_bounds, upper_bounds def check_memory(map_data, ratio_needed, maximum_fraction_to_use = 0.90, maximum_map_size = 1, out = sys.stdout): map_size = map_data.size()/(1024*1024*1024) if maximum_map_size and map_size>maximum_map_size: raise Sorry("Maximum map size for this tool is %s GB" %(maximum_map_size)) needed_memory = ratio_needed*map_size from libtbx.utils import guess_total_memory # returns total memory bytes_total_memory = guess_total_memory() if bytes_total_memory: total_memory = bytes_total_memory/(1024*1024*1024) else: total_memory = None print("\nMap size is " +\ "%.2f GB. This will require about %.1f GB of memory" %( map_size, needed_memory) +"\nfor this stage of analysis\n", file = out) if total_memory: print("Total memory on this computer is about %.1f GB." %( total_memory), file = out) if (needed_memory>= 0.5* total_memory): print("\n ** WARNING: It is possible that this computer may not"+\ " have **\n *** sufficient memory to complete this job. ***\n", file = out) if (needed_memory >= maximum_fraction_to_use*total_memory): raise Sorry("This computer does not have sufficient "+ "memory (%.0f GB needed) \nto run this job" %(needed_memory)) def get_params(args, map_data = None, crystal_symmetry = None, half_map_data_list = None, sharpening_target_pdb_inp = None, ncs_object = None, write_files = None, auto_sharpen = None, density_select = None, add_neighbors = None, save_box_map_ncs_au = None, sequence = None, wrapping = None, target_ncs_au_file = None, regions_to_keep = None, solvent_content = None, resolution = None, molecular_mass = None, symmetry = None, chain_type = None, keep_low_density = None, box_buffer = None, soft_mask_extract_unique = None, mask_expand_ratio = None, include_helical_symmetry = None, min_ncs_cc = None, symmetry_center = None, return_params_only = None, out = sys.stdout): params = get_params_from_args(args) # Set params specifically if coming in from call if sequence is not None: params.crystal_info.sequence = sequence if (wrapping is not None) and (params.crystal_info.use_sg_symmetry is None): params.crystal_info.use_sg_symmetry = wrapping if target_ncs_au_file is not None: params.input_files.target_ncs_au_file = target_ncs_au_file if regions_to_keep is not None: params.map_modification.regions_to_keep = regions_to_keep if solvent_content is not None: params.crystal_info.solvent_content = solvent_content if resolution is not None: params.crystal_info.resolution = resolution if molecular_mass is not None: params.crystal_info.molecular_mass = molecular_mass if symmetry is not None: params.reconstruction_symmetry.symmetry = symmetry if min_ncs_cc is not None: params.reconstruction_symmetry.min_ncs_cc = min_ncs_cc if symmetry_center is not None: params.reconstruction_symmetry.symmetry_center = symmetry_center if include_helical_symmetry is not None: params.reconstruction_symmetry.include_helical_symmetry = \ include_helical_symmetry if chain_type is not None: params.crystal_info.chain_type = chain_type if regions_to_keep is not None: params.map_modification.regions_to_keep = regions_to_keep params.segmentation.iterate_with_remainder = False elif keep_low_density: params.segmentation.iterate_with_remainder = True elif keep_low_density is False: params.segmentation.iterate_with_remainder = False else: pass # just so it is clear this was considered if box_buffer is not None: params.output_files.box_buffer = box_buffer if soft_mask_extract_unique is not None: params.map_modification.soft_mask = soft_mask_extract_unique if mask_expand_ratio is not None: params.segmentation.mask_expand_ratio = mask_expand_ratio if write_files is not None: params.control.write_files = write_files if auto_sharpen is not None: params.map_modification.auto_sharpen = auto_sharpen if density_select is not None: params.segmentation.density_select = density_select if add_neighbors is not None: params.segmentation.add_neighbors = add_neighbors if save_box_map_ncs_au is not None: params.control.save_box_map_ncs_au = save_box_map_ncs_au if return_params_only: return params print("\nSegment_and_split_map\n", file = out) print("Command used: %s\n" %( " ".join(['segment_and_split_map']+args)), file = out) master_params.format(python_object = params).show(out = out) # Set space-group defaults if params.crystal_info.use_sg_symmetry: if params.map_modification.restrict_map_size is None: params.map_modification.restrict_map_size = False if params.crystal_info.is_crystal is None: params.crystal_info.is_crystal = True else: if params.map_modification.restrict_map_size is None: params.map_modification.restrict_map_size = True if params.crystal_info.is_crystal is None: params.crystal_info.is_crystal = False # Turn off files if desired if params.control.write_files is False: params.output_files.magnification_map_file = None params.output_files.magnification_map_file = None params.output_files.magnification_ncs_file = None params.output_files.shifted_map_file = None params.output_files.shifted_sharpened_map_file = None params.output_files.sharpened_map_file = None params.output_files.shifted_pdb_file = None params.output_files.shifted_ncs_file = None params.output_files.shifted_used_ncs_file = None params.output_files.box_map_file = None params.output_files.box_mask_file = None params.output_files.write_output_maps = False params.output_files.remainder_map_file = None params.output_files.output_info_file = None params.output_files.restored_pdb = None params.output_files.output_weight_map_pickle_file = None from cctbx.maptbx.auto_sharpen import set_sharpen_params params = set_sharpen_params(params, out) if params.input_files.seq_file and not params.crystal_info.sequence and \ not sequence: if not params.crystal_info.sequence: if sequence: params.crystal_info.sequence = sequence else: params.crystal_info.sequence = open(params.input_files.seq_file).read() print("Read sequence from %s" %(params.input_files.seq_file), file = out) if not params.crystal_info.resolution and ( params.map_modification.b_iso is not None or \ params.map_modification.auto_sharpen or params.map_modification.resolution_dependent_b or params.map_modification.b_sharpen): raise Sorry("Need resolution for segment_and_split_map with sharpening") if params.map_modification.auto_sharpen and ( params.map_modification.b_iso is not None or params.map_modification.b_sharpen is not None or params.map_modification.resolution_dependent_b is not None): print("Turning off auto_sharpen as it is not compatible with "+\ "b_iso, \nb_sharpen, or resolution_dependent_b", file = out) params.map_modification.auto_sharpen = False if params.control.write_files and \ params.output_files.output_directory and \ (not os.path.isdir(params.output_files.output_directory)): os.mkdir(params.output_files.output_directory) if not params.output_files.output_directory: params.output_files.output_directory = "" # Test to see if we can use adjusted_sa as target and use box_map with it if (params.map_modification.residual_target == 'adjusted_sa' or params.map_modification.sharpening_target == 'adjusted_sa') and \ (params.map_modification.box_in_auto_sharpen or params.map_modification.density_select_in_auto_sharpen) and ( params.map_modification.auto_sharpen): print("Checking to make sure we can use adjusted_sa as target...", end = ' ', file = out) try: from phenix.autosol.map_to_model import iterated_solvent_fraction dummy = iterated_solvent_fraction # just to import it except Exception as e: raise Sorry("Please either set box_in_auto_sharpen = False and "+ "\ndensity_select_in_auto_sharpen = False or \n"+\ "set residual_target = kurtosis and sharpening_target = kurtosis") print("OK", file = out) if not half_map_data_list: half_map_data_list = [] if params.input_files.info_file: map_data = None pdb_hierarchy = None from libtbx import easy_pickle print("Loading tracking data from %s" %( params.input_files.info_file), file = out) tracking_data = easy_pickle.load(params.input_files.info_file) return params, map_data, half_map_data_list, pdb_hierarchy, tracking_data, None else: tracking_data = info_object() tracking_data.set_params(params) # PDB file if params.input_files.pdb_file: print("\nInput PDB file to be used to identify region to work with: %s\n" %( params.input_files.pdb_file), file = out) pdb_inp = iotbx.pdb.input(file_name = params.input_files.pdb_file) pdb_hierarchy = pdb_inp.construct_hierarchy() pdb_atoms = pdb_hierarchy.atoms() pdb_atoms.reset_i_seq() tracking_data.set_input_pdb_info(file_name = params.input_files.pdb_file, n_residues = pdb_hierarchy.overall_counts().n_residues) else: pdb_hierarchy = None if map_data: pass # ok elif params.input_files.map_file: ccp4_map = iotbx.mrcfile.map_reader( file_name = params.input_files.map_file) if not crystal_symmetry: crystal_symmetry = ccp4_map.crystal_symmetry() # 2018-07-18 tracking_data.set_full_crystal_symmetry( ccp4_map.unit_cell_crystal_symmetry()) tracking_data.set_full_unit_cell_grid(ccp4_map.unit_cell_grid) map_data = ccp4_map.data.as_double() else: raise Sorry("Need ccp4 map") if not crystal_symmetry: raise Sorry("Need crystal_symmetry") if params.input_files.half_map_file: if len(params.input_files.half_map_file) != 2: raise Sorry("Please supply none or two half_map_file values") half_map_data_list = [] half_map_data_list.append(iotbx.mrcfile.map_reader( file_name = params.input_files.half_map_file[0]).data.as_double()) half_map_data_list.append(iotbx.mrcfile.map_reader( file_name = params.input_files.half_map_file[1]).data.as_double()) # Get the NCS object ncs_obj, dummy_tracking_data = get_ncs(params = params, ncs_object = ncs_object, out = out) if (not params.map_modification.auto_sharpen or params.map_modification.b_iso is not None) and ( not params.crystal_info.molecular_mass and not params.crystal_info.solvent_content and not params.input_files.seq_file and not params.crystal_info.sequence and not sequence): params.crystal_info.solvent_content = get_iterated_solvent_fraction( crystal_symmetry = crystal_symmetry, verbose = params.control.verbose, resolve_size = params.control.resolve_size, mask_padding_fraction = \ params.segmentation.mask_padding_fraction, fraction_of_max_mask_threshold = \ params.segmentation.fraction_of_max_mask_threshold, cell_cutoff_for_solvent_from_mask = \ params.segmentation.cell_cutoff_for_solvent_from_mask, mask_resolution = params.crystal_info.resolution, map = map_data, out = out) if params.crystal_info.solvent_content: print("Estimated solvent content: %.2f" %( params.crystal_info.solvent_content), file = out) else: raise Sorry("Unable to estimate solvent content...please supply "+ "solvent_content \nor molecular_mass") if params.map_modification.auto_sharpen or \ params.map_modification.b_iso is not None or \ params.map_modification.b_sharpen is not None or \ params.map_modification.resolution_dependent_b is not None: # Sharpen the map print("Auto-sharpening map before using it", file = out) local_params = deepcopy(params) if tracking_data.solvent_fraction: # XXX was previously always done but may not have been set local_params.crystal_info.solvent_content = tracking_data.solvent_fraction from cctbx.maptbx.auto_sharpen import run as auto_sharpen acc = map_data.accessor() map_data, new_map_coeffs, new_crystal_symmetry, new_si = auto_sharpen( args = [], params = local_params, map_data = map_data, wrapping = wrapping, crystal_symmetry = crystal_symmetry, write_output_files = False, pdb_inp = sharpening_target_pdb_inp, ncs_obj = ncs_obj, return_map_data_only = False, return_unshifted_map = True, half_map_data_list = half_map_data_list, n_residues = tracking_data.n_residues, ncs_copies = ncs_obj.max_operators(), out = out) tracking_data.b_sharpen = new_si.b_sharpen if not tracking_data.solvent_fraction: tracking_data.solvent_fraction = new_si.solvent_fraction if tracking_data.params.output_files.sharpened_map_file: sharpened_map_file = os.path.join( tracking_data.params.output_files.output_directory, tracking_data.params.output_files.sharpened_map_file) sharpened_map_data = map_data.deep_copy() if acc is not None: # offset the map to match original if possible sharpened_map_data.reshape(acc) print("Gridding of sharpened map:", file = out) print("Origin: ", sharpened_map_data.origin(), file = out) print("All: ", sharpened_map_data.all(), file = out) print("\nWrote sharpened map in original location with "+\ "origin at %s\nto %s" %( str(sharpened_map_data.origin()), sharpened_map_file), file = out) # NOTE: original unit cell and grid write_ccp4_map(tracking_data.full_crystal_symmetry, sharpened_map_file, sharpened_map_data, output_unit_cell_grid = tracking_data.full_unit_cell_grid, ) params.input_files.map_file = sharpened_map_file # overwrite map_file name here # done with any sharpening params.map_modification.auto_sharpen = False# so we don't do it again later params.map_modification.b_iso = None params.map_modification.b_sharpen = None params.map_modification.resolution_dependent_b = None if params.control.sharpen_only: print("Stopping after sharpening", file = out) return # check on size right away if params.control.memory_check: # map_box and mask generation use about 50GB of memory for # map with 1 billion elements check_memory(map_data = map_data, maximum_map_size = None, ratio_needed = 50, out = out) if params.map_modification.magnification and \ params.map_modification.magnification!= 1.0: print("\nAdjusting magnification by %7.3f\n" %( params.map_modification.magnification), file = out) if ncs_obj: # Magnify ncs print("NCS before applying magnification...", file = out) ncs_obj.format_all_for_group_specification(out = out) ncs_obj = ncs_obj.adjust_magnification( magnification = params.map_modification.magnification) if params.output_files.magnification_ncs_file: file_name = os.path.join(params.output_files.output_directory, params.output_files.magnification_ncs_file) print("Writing NCS after magnification of %7.3f to %s" %( params.map_modification.magnification, file_name), file = out) ncs_obj.format_all_for_group_specification(out = out) ncs_obj.format_all_for_group_specification(file_name = file_name) params.input_files.ncs_file = file_name else: raise Sorry("Need magnification_ncs_file defined if magnification is"+ " applied \nto input NCS file") # Magnify map shrunk_uc = [] for i in range(3): shrunk_uc.append( crystal_symmetry.unit_cell().parameters()[i] * params.map_modification.magnification ) uc_params = crystal_symmetry.unit_cell().parameters() from cctbx import uctbx new_unit_cell = uctbx.unit_cell( parameters = (shrunk_uc[0], shrunk_uc[1], shrunk_uc[2], uc_params[3], uc_params[4], uc_params[5])) print("Original unit cell: (%7.4f, %7.4f, %7.4f, %7.4f, %7.4f, %7.4f)" %( crystal_symmetry.unit_cell().parameters()), file = out) crystal_symmetry = crystal.symmetry( unit_cell = new_unit_cell, space_group = crystal_symmetry.space_group()) print("New unit cell: (%7.4f, %7.4f, %7.4f, %7.4f, %7.4f, %7.4f)" %( crystal_symmetry.unit_cell().parameters()), file = out) # magnify original unit cell too.. cell = list(tracking_data.full_crystal_symmetry.unit_cell().parameters()) for i in range(3): cell[i] = cell[i]*params.map_modification.magnification tracking_data.set_full_crystal_symmetry( crystal.symmetry(tuple(cell), ccp4_map.space_group_number)) print("New original (full unit cell): "+\ " (%7.4f, %7.4f, %7.4f, %7.4f, %7.4f, %7.4f)" %( tracking_data.full_crystal_symmetry.unit_cell.parameters()), file = out) if params.output_files.magnification_map_file: file_name = os.path.join(params.output_files.output_directory, params.output_files.magnification_map_file) # write out magnified map (our working map) (before shifting it) print("\nWriting magnification map (input map with "+\ "magnification of %7.3f \n" %(params.map_modification.magnification) +\ "applied) to %s \n" %(file_name), file = out) #write_ccp4_map(crystal_symmetry, file_name, map_data) # NOTE: original unit cell and grid write_ccp4_map(tracking_data.full_crystal_symmetry, file_name, map_data, output_unit_cell_grid = tracking_data.original_unit_cell_grid, ) params.input_files.map_file = file_name else: raise Sorry("Need a file name to write out magnification_map_file") params.map_modification.magnification = None # no longer need it. tracking_data.set_input_map_info(file_name = params.input_files.map_file, crystal_symmetry = crystal_symmetry, origin = map_data.origin(), all = map_data.all()) tracking_data.set_crystal_symmetry(crystal_symmetry = crystal_symmetry) tracking_data.set_original_crystal_symmetry(crystal_symmetry = crystal_symmetry) tracking_data.set_accessor(acc = map_data.accessor()) # Save center of map map_symmetry_center = get_center_of_map(map_data, crystal_symmetry) # Check for helical ncs...if present we may try to cut map at +/- 1 turn params.map_modification.restrict_z_distance_for_helical_symmetry = \ get_max_z_range_for_helical_symmetry(params, out = out) # either use map_box with density_select = True or just shift the map if params.segmentation.density_select: print("\nTrimming map to density...", file = out) args = ["output_format = ccp4"] if params.segmentation.density_select_threshold is not None: print("Threshold for density selection will be: %6.2f \n"%( params.segmentation.density_select_threshold), file = out) args.append("density_select_threshold = %s" %( params.segmentation.density_select_threshold)) if params.segmentation.get_half_height_width is not None: args.append("get_half_height_width = %s" %( params.segmentation.get_half_height_width)) if params.input_files.ncs_file: args.append("symmetry_file = %s" %(params.input_files.ncs_file)) if params.input_files.pdb_file: args.append("pdb_file = %s" %(params.input_files.pdb_file)) args.append("ccp4_map_file = %s" %(params.input_files.map_file)) file_name_prefix = os.path.join(params.output_files.output_directory, "density_select") args.append("output_file_name_prefix = %s" %(file_name_prefix)) from mmtbx.command_line.map_box import run as run_map_box args.append("keep_input_unit_cell_and_grid = False") # for new defaults assert params.crystal_info.use_sg_symmetry is not None wrapping = params.crystal_info.use_sg_symmetry if params.segmentation.lower_bounds and params.segmentation.upper_bounds: bounds_supplied = True print("\nRunning map_box with supplied bounds", file = out) box = run_map_box(args, map_data = map_data, ncs_object = ncs_obj, crystal_symmetry = crystal_symmetry, lower_bounds = params.segmentation.lower_bounds, upper_bounds = params.segmentation.upper_bounds, write_output_files = params.output_files.write_output_maps, wrapping = wrapping, log = out) else: bounds_supplied = False box = run_map_box(["density_select = True"]+args, map_data = map_data, crystal_symmetry = crystal_symmetry, ncs_object = ncs_obj, write_output_files = params.output_files.write_output_maps, wrapping = wrapping, log = out) # Run again to select au box shifted_unique_closest_sites = None selected_au_box = None if params.segmentation.select_au_box is None and box.ncs_object and \ box.ncs_object.max_operators() >= params.segmentation.n_ops_to_use_au_box: params.segmentation.select_au_box = True print("Setting select_au_box to True as there are %d operators" %( box.ncs_object.max_operators()), file = out) if params.segmentation.select_au_box and not bounds_supplied: lower_bounds, upper_bounds, unique_closest_sites = get_bounds_for_au_box( params, box = box, out = out) #unique_closest_sites relative to original map if lower_bounds and upper_bounds: bounds_supplied = True selected_au_box = True score, ncs_cc = score_ncs_in_map(map_data = box.map_box, allow_score_with_pg = False, sites_orth = unique_closest_sites+box.shift_cart, ncs_object = box.ncs_object, ncs_in_cell_only = True, crystal_symmetry = box.box_crystal_symmetry, out = null_out()) print("NCS CC before rerunning box: %7.2f SCORE: %7.1f OPS: %d " %( ncs_cc, score, box.ncs_object.max_operators()), file = out) print("\nRunning map-box again with boxed range ...", file = out) del box box = run_map_box(args, lower_bounds = lower_bounds, map_data = map_data, crystal_symmetry = crystal_symmetry, ncs_object = ncs_obj, wrapping = wrapping, upper_bounds = upper_bounds, log = out) box.map_box = box.map_box.as_double() # Do we need double? shifted_unique_closest_sites = unique_closest_sites+box.shift_cart # Or run again for helical symmetry elif params.map_modification.restrict_z_distance_for_helical_symmetry and \ not bounds_supplied: bounds_supplied = True lower_bounds, upper_bounds = get_bounds_for_helical_symmetry(params, crystal_symmetry = crystal_symmetry, box = box) print("\nRunning map-box again with restricted Z range ...", file = out) box = run_map_box(args, map_data = map_data, crystal_symmetry = crystal_symmetry, ncs_object = ncs_obj, lower_bounds = lower_bounds, upper_bounds = upper_bounds, wrapping = wrapping, write_output_files = params.output_files.write_output_maps, log = out) #----------------------------- if bounds_supplied and box.ncs_object: print("Selecting remaining NCS operators", file = out) box.ncs_object = select_remaining_ncs_ops( map_data = box.map_box, crystal_symmetry = box.box_crystal_symmetry, closest_sites = shifted_unique_closest_sites, random_points = params.reconstruction_symmetry.random_points, ncs_object = box.ncs_object, out = out) score, ncs_cc = score_ncs_in_map(map_data = box.map_box, allow_score_with_pg = False, ncs_object = box.ncs_object, ncs_in_cell_only = True, sites_orth = shifted_unique_closest_sites, crystal_symmetry = box.box_crystal_symmetry, out = null_out()) if score is not None: print("NCS CC after selections: %7.2f SCORE: %7.1f OPS: %d" %( ncs_cc, score, box.ncs_object.max_operators()), file = out) #----------------------------- origin_shift = box.shift_cart # Note: moving cell with (0, 0, 0) in middle to (0, 0, 0) at corner means # total_shift_cart and origin_shift both positive map_data = box.map_box map_data = scale_map(map_data, out = out) crystal_symmetry = box.box_crystal_symmetry print("New unit cell: %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f " %( crystal_symmetry.unit_cell().parameters()), file = out) tracking_data.set_crystal_symmetry(crystal_symmetry = crystal_symmetry) print("Moving origin to (0, 0, 0)", file = out) print("Adding (%8.2f, %8.2f, %8.2f) to all coordinates\n"%( tuple(origin_shift)), file = out) # NOTE: size and cell params are now different! tracking_data.set_box_map_bounds_first_last( box.gridding_first, box.gridding_last) new_half_map_data_list = [] ii = 0 for hm in half_map_data_list: ii+= 1 hm = hm.shift_origin() # shift if necessary hm = box.cut_and_copy_map(map_data = hm).as_double() hm.reshape(flex.grid(hm.all())) new_half_map_data_list.append(hm) cutout_half_map_file = os.path.join(params.output_files.output_directory, "cutout_half_map_%s.ccp4" %(ii)) print("Writing cutout half_map data to %s" %(cutout_half_map_file), file = out) write_ccp4_map(crystal_symmetry, cutout_half_map_file, new_half_map_data_list[-1]) half_map_data_list = new_half_map_data_list if params.map_modification.soft_mask: mask_data, map_data, half_map_data_list, \ soft_mask_solvent_fraction, smoothed_mask_data, \ original_box_map_data = \ get_and_apply_soft_mask_to_maps( resolution = params.crystal_info.resolution, wang_radius = params.crystal_info.wang_radius, buffer_radius = params.crystal_info.buffer_radius, map_data = map_data, crystal_symmetry = crystal_symmetry, half_map_data_list = half_map_data_list, out = out) print("\nSolvent fraction from soft mask procedure: %7.2f (not used)\n" %( soft_mask_solvent_fraction), file = out) shifted_ncs_object = box.ncs_object if not shifted_ncs_object or shifted_ncs_object.max_operators()<2: from mmtbx.ncs.ncs import ncs shifted_ncs_object = ncs() shifted_ncs_object.set_unit_ncs() else: # shift if necessary... shift_needed = not \ (map_data.focus_size_1d() > 0 and map_data.nd() == 3 and map_data.is_0_based()) a, b, c = crystal_symmetry.unit_cell().parameters()[:3] N_ = map_data.all() O_ = map_data.origin() sx, sy, sz = O_[0]/N_[0], O_[1]/N_[1], O_[2]/N_[2] # Note: If (0, 0, 0) is in the middle of the box, origin at sx, sy, sz # is negative, shift of coordinates will be positive sx_cart, sy_cart, sz_cart = crystal_symmetry.unit_cell().orthogonalize( [sx, sy, sz]) print("Origin for input map is at (%8.2f, %8.2f, %8.2f)" % ( sx_cart, sy_cart, sz_cart), file = out) print("Cell dimensions of this map are: (%8.2f, %8.2f, %8.2f)" % (a, b, c), file = out) if shift_needed: if(not crystal_symmetry.space_group().type().number() in [0, 1]): raise RuntimeError("Not implemented") origin_shift = [-sx_cart, -sy_cart, -sz_cart] # positive if (0, 0, 0) in middle print("Adding (%8.2f, %8.2f, %8.2f) to all coordinates"%( tuple(origin_shift))+" to put origin at (0, 0, 0)\n", file = out) map_data = map_data.shift_origin() new_half_map_data_list = [] for hm in half_map_data_list: new_half_map_data_list.append(hm.shift_origin()) half_map_data_list = new_half_map_data_list else: origin_shift = (0., 0., 0.) # Get NCS object if any if params.input_files.ncs_file and not ncs_obj: ncs_obj, dummy_obj = get_ncs(file_name = params.input_files.ncs_file) if ncs_obj: shifted_ncs_object = ncs_obj.coordinate_offset( coordinate_offset = matrix.col(origin_shift)) # shift to match shifted map else: from mmtbx.ncs.ncs import ncs shifted_ncs_object = ncs() shifted_ncs_object.set_unit_ncs() update_tracking_data_with_sharpening( map_data = map_data, tracking_data = tracking_data, out = out) # Set origin shift now tracking_data.set_origin_shift(origin_shift) map_symmetry_center = matrix.col(map_symmetry_center)+matrix.col(origin_shift) # New ctr if shifted_ncs_object and params.control.check_ncs: ncs_obj_to_check = shifted_ncs_object else: ncs_obj_to_check = None found_ncs = False if params.reconstruction_symmetry.symmetry or ncs_obj_to_check or \ params.reconstruction_symmetry.optimize_center: looking_for_ncs = True new_ncs_obj, ncs_cc, ncs_score = run_get_ncs_from_map(params = params, map_data = map_data, map_symmetry_center = map_symmetry_center, crystal_symmetry = crystal_symmetry, ncs_obj = ncs_obj_to_check, out = out, ) if new_ncs_obj: found_ncs = True shifted_ncs_object = new_ncs_obj.deep_copy() # offset this back to where it would have been before the origin offset.. new_ncs_obj = new_ncs_obj.coordinate_offset( coordinate_offset = -1*matrix.col(origin_shift)) # XXX save it in tracking_data if params.output_files.output_directory: if not os.path.isdir(params.output_files.output_directory): os.mkdir(params.output_files.output_directory) file_name = os.path.join(params.output_files.output_directory, 'ncs_from_map.ncs_spec') f = open(file_name, 'w') new_ncs_obj.format_all_for_group_specification(out = f) f.close() print("Wrote NCS operators (for original map) to %s" %(file_name), file = out) if not params.control.check_ncs: params.input_files.ncs_file = file_name # set it else: looking_for_ncs = False if params.control.check_ncs: print("Done checking NCS", file = out) return params, map_data, half_map_data_list, pdb_hierarchy, tracking_data, None if looking_for_ncs and (not found_ncs) and \ params.reconstruction_symmetry.symmetry.upper() not in ['ANY', 'ALL']: raise Sorry( "Unable to identify %s symmetry automatically in this map." %( params.reconstruction_symmetry.symmetry)+ "\nPlease supply a symmetry file with symmetry matrices.") if params.segmentation.expand_size is None: params.segmentation.expand_size = estimate_expand_size( crystal_symmetry = crystal_symmetry, map_data = map_data, expand_target = params.segmentation.expand_target, out = out) if params.output_files.output_info_file and params.control.shift_only: write_info_file(params = params, tracking_data = tracking_data, out = out) return params, map_data, half_map_data_list, pdb_hierarchy, \ tracking_data, shifted_ncs_object def write_info_file(params = None, tracking_data = None, out = sys.stdout): # write out the info file from libtbx import easy_pickle tracking_data.show_summary(out = out) print("\nWriting summary information to: %s" %( os.path.join(tracking_data.params.output_files.output_directory, params.output_files.output_info_file)), file = out) print("\nTo restore original position of a PDB file built into these maps, use:", file = out) print("phenix.segment_and_split_map info_file = %s" %( os.path.join(tracking_data.params.output_files.output_directory, params.output_files.output_info_file))+" pdb_to_restore = mypdb.pdb\n", file = out) easy_pickle.dump(os.path.join(tracking_data.params.output_files.output_directory, params.output_files.output_info_file), tracking_data) def get_and_apply_soft_mask_to_maps( resolution = None, #params.crystal_info.resolution wang_radius = None, #params.crystal_info.wang_radius buffer_radius = None, #params.crystal_info.buffer_radius force_buffer_radius = None, # apply buffer radius always map_data = None, crystal_symmetry = None, solvent_content = None, solvent_content_iterations = None, return_masked_fraction = True, rad_smooth = None, half_map_data_list = None, out = sys.stdout): smoothed_mask_data = None if not resolution: from cctbx.maptbx import d_min_from_map resolution = d_min_from_map( map_data, crystal_symmetry.unit_cell(), resolution_factor = 1./4.) if not rad_smooth: rad_smooth = resolution if rad_smooth: print("\nApplying soft mask with smoothing radius of %.2f A\n" %( rad_smooth), file = out) if wang_radius: wang_radius = wang_radius else: wang_radius = 1.5*resolution if buffer_radius is not None: buffer_radius = buffer_radius else: buffer_radius = 2.*resolution original_map_data = map_data.deep_copy() # Check to make sure this is possible cell_dims = crystal_symmetry.unit_cell().parameters()[:3] min_cell_dim = min(cell_dims) if wang_radius > 0.25 * min_cell_dim or buffer_radius > 0.25 * min_cell_dim: new_wang_radius = min(wang_radius, 0.25 * min_cell_dim) new_buffer_radius = min(buffer_radius, 0.25 * min_cell_dim) print ("Cell is too small to get solvent fraction ...resetting "+ "values of wang_radius \n"+ "(was %.3f A now %.3f A) and buffer_radius (was %.3f A now %.3f A)" %( wang_radius, new_wang_radius, buffer_radius, new_buffer_radius), file = out) wang_radius = new_wang_radius buffer_radius = new_buffer_radius mask_data, solvent_fraction = get_mask_around_molecule(map_data = map_data, crystal_symmetry = crystal_symmetry, wang_radius = wang_radius, solvent_content = solvent_content, solvent_content_iterations = solvent_content_iterations, buffer_radius = buffer_radius, force_buffer_radius = force_buffer_radius, return_masked_fraction = return_masked_fraction, out = out) if mask_data: map_data, smoothed_mask_data = apply_soft_mask(map_data = map_data, mask_data = mask_data.as_double(), rad_smooth = rad_smooth, crystal_symmetry = crystal_symmetry, out = out) new_half_map_data_list = [] if not half_map_data_list: half_map_data_list = [] for half_map in half_map_data_list: assert half_map.size() == mask_data.size() half_map, smoothed_mask_data = apply_soft_mask(map_data = half_map, mask_data = mask_data.as_double(), rad_smooth = rad_smooth, crystal_symmetry = crystal_symmetry, out = out) new_half_map_data_list.append(half_map) half_map_data_list = new_half_map_data_list else: print("Unable to get mask...skipping", file = out) return mask_data, map_data, half_map_data_list, \ solvent_fraction, smoothed_mask_data, original_map_data def get_ncs(params = None, tracking_data = None, file_name = None, ncs_object = None, out = sys.stdout): if not file_name: file_name = params.input_files.ncs_file if (not ncs_object or ncs_object.max_operators()<2) and file_name: print("Reading ncs from %s" %(file_name), file = out) is_helical_symmetry = None if (not ncs_object or ncs_object.max_operators()<2) and not file_name: # No ncs supplied...use just 1 ncs copy.. from mmtbx.ncs.ncs import ncs ncs_object = ncs() ncs_object.set_unit_ncs() #ncs_object.display_all(log = out) elif (not ncs_object or ncs_object.max_operators()<2) and \ not os.path.isfile(file_name): raise Sorry("The ncs file %s is missing" %(file_name)) else: # get the ncs if not ncs_object: from mmtbx.ncs.ncs import ncs ncs_object = ncs() try: # see if we can read biomtr records pdb_inp = iotbx.pdb.input(file_name = file_name) ncs_object.ncs_from_pdb_input_BIOMT(pdb_inp = pdb_inp, log = out) except Exception as e: # try as regular ncs object ncs_object.read_ncs(file_name = file_name, log = out) #ncs_object.display_all(log = out) ncs_object.select_first_ncs_group() if ncs_object.max_operators()<1: from mmtbx.ncs.ncs import ncs ncs_object = ncs() ncs_object.set_unit_ncs() print("\nTotal of %d NCS operators read\n" %( ncs_object.max_operators()), file = out) if not tracking_data or not params: return ncs_object, None if ncs_object.max_operators()<2: print("No NCS present", file = out) elif ncs_object.is_helical_along_z( abs_tol_t = tracking_data.params.reconstruction_symmetry.abs_tol_t, rel_tol_t = tracking_data.params.reconstruction_symmetry.rel_tol_t, tol_r = tracking_data.params.reconstruction_symmetry.tol_r): print("This NCS is helical symmetry", file = out) is_helical_symmetry = True elif ncs_object.is_point_group_symmetry( abs_tol_t = tracking_data.params.reconstruction_symmetry.abs_tol_t, rel_tol_t = tracking_data.params.reconstruction_symmetry.rel_tol_t, tol_r = tracking_data.params.reconstruction_symmetry.tol_r): print("This NCS is point-group symmetry", file = out) elif params.crystal_info.is_crystal: print("This NCS is crystal symmetry", file = out) elif not ( params.reconstruction_symmetry.require_helical_or_point_group_symmetry): print("WARNING: NCS is not crystal symmetry nor point-group "+\ "symmetry nor helical symmetry", file = out) else: raise Sorry("Need point-group or helical symmetry.") if not ncs_object or ncs_object.max_operators()<1: raise Sorry("Need ncs information from an ncs_info file") if tracking_data: tracking_data.set_input_ncs_info(file_name = file_name, # XXX may be updated ops number_of_operators = ncs_object.max_operators()) if tracking_data and is_helical_symmetry: # update shifted_ncs_info if tracking_data.shifted_ncs_info: # XXX may not be needed shifted = True else: shifted = False print("Updating NCS info (shifted = %s)" %(shifted), file = out) tracking_data.update_ncs_info(is_helical_symmetry = True, shifted = shifted) if tracking_data.input_map_info and tracking_data.input_map_info.all: z_range = tracking_data.crystal_symmetry.unit_cell(). \ parameters()[2] print("Extending NCS operators to entire cell (z_range = %.1f)" %( z_range), file = out) max_operators = \ tracking_data.params.reconstruction_symmetry.max_helical_operators if max_operators: print("Maximum new number of NCS operators will be %s" %( max_operators), file = out) ncs_object.extend_helix_operators(z_range = z_range, max_operators = max_operators) #ncs_object.display_all() print("New number of NCS operators is: %s " %( ncs_object.max_operators()), file = out) tracking_data.update_ncs_info( number_of_operators = ncs_object.max_operators(), is_helical_symmetry = True, shifted = shifted) return ncs_object, tracking_data def score_threshold(b_vs_region = None, threshold = None, sorted_by_volume = None, n_residues = None, ncs_copies = None, fraction_occupied = None, solvent_fraction = None, map_data = None, residues_per_region = 50, min_volume = None, min_ratio = None, max_ratio_to_target = None, min_ratio_to_target = None, weight_score_grid_points = 1., weight_score_ratio = 1.0, weight_near_one = 0.1, min_ratio_of_ncs_copy_to_first = None, target_in_all_regions = None, crystal_symmetry = None, chain_type = None, out = sys.stdout): # We want about 1 region per 50-100 residues for the biggest region. # One possibility is to try to maximize the median size of the N top # regions, where N = number of expected regions = n_residues/residues_per_region # Also note we have an idea how big a region should be (how many # grid points) if we make an assumption about the fractional volume that # should be inside a region compared to the total volume of protein/nucleic # acid in the region...this gives us target_in_top_regions points. # So using this, make the median size as close to target_in_top_regions as # we can. # If we have solvent fraction but not ncs_copies or n_residues, guess the # number of residues and ncs copies from the volume if ncs_copies is not None and n_residues is not None: expected_regions = max(ncs_copies, max(1, int(0.5+n_residues/residues_per_region))) else: if chain_type in [None, 'None']: chain_type = "PROTEIN" assert crystal_symmetry is not None assert solvent_fraction is not None volume_per_residue, nres, chain_type = get_volume_of_seq( "A", chain_type = chain_type, out = out) expected_regions = max(1, int(0.5+(1-solvent_fraction)*\ crystal_symmetry.unit_cell().volume()/volume_per_residue )) # NOTE: This is expected residues. expected_regions should be this # divided by residues_per_region expected_regions = max(1, int(0.5+expected_regions/residues_per_region)) ncs_copies = 1 target_in_top_regions = target_in_all_regions/expected_regions nn = len(sorted_by_volume)-1 # first one is total ok = True too_low = None # marker for way too low too_high = None if nn < ncs_copies: ok = False #return # not enough v1, i1 = sorted_by_volume[1] if v1 < min_volume: ok = False #return if v1 > max_ratio_to_target*target_in_top_regions: ok = False #return too_low = True if v1 < min_volume or v1 < 0.1*min_ratio_to_target*target_in_top_regions: # way too high too_high = True # there should be about ncs_copies copies of each size region if ncs_copies>1 if ncs_copies>1: v2, i2 = sorted_by_volume[max(1, min(ncs_copies, nn))] score_ratio = v2/v1 # want it to be about 1 if score_ratio < min_ratio_of_ncs_copy_to_first: ok = False #return # not allowed else: score_ratio = 1.0 # for ncs_copies = 1 nn2 = min(nn, max(1, (expected_regions+1)//2)) median_number, iavg = sorted_by_volume[nn2] # number in each region should be about target_in_top_regions if median_number > target_in_top_regions: score_grid_points = target_in_top_regions/max(1., median_number) else: score_grid_points = median_number/target_in_top_regions if v1> target_in_top_regions: score_grid_points_b = target_in_top_regions/max(1., v1) else: score_grid_points_b = v1/target_in_top_regions score_grid_points = 0.5*(score_grid_points+score_grid_points_b) score_grid_points = score_grid_points**2 # maybe even **3 if threshold>1.: score_near_one = 1./threshold else: score_near_one = threshold # Normalize weight_score_ratio by target_in_top_regions: sc = min(1., 0.5*median_number/max(1, target_in_top_regions)) overall_score = ( (sc*weight_score_ratio*score_ratio+ weight_score_grid_points*score_grid_points+ weight_near_one*score_near_one ) / (weight_score_ratio+weight_score_grid_points+weight_near_one)) half_expected_regions = max(1, (1+expected_regions)//2) ratio = sorted_by_volume[min(len(sorted_by_volume)-1, half_expected_regions)][0]/v1 if ok and v1 >= target_in_top_regions/2 and \ len(sorted_by_volume)>half_expected_regions: last_volume = sorted_by_volume[half_expected_regions][0] if ratio >= min_ratio and \ last_volume>= min_volume: has_sufficient_regions = True else: has_sufficient_regions = False else: has_sufficient_regions = False print("%7.2f %5.2f %5d %4d %5d %5d %6.3f %5s %5.3f %s %s" %( b_vs_region.b_iso, threshold, target_in_top_regions, expected_regions, v1, median_number, ratio, has_sufficient_regions, overall_score, ok, nn), file = out) if not b_vs_region.b_iso in b_vs_region.b_vs_region_dict.keys(): b_vs_region.b_vs_region_dict[b_vs_region.b_iso] = {} b_vs_region.sa_sum_v_vs_region_dict[b_vs_region.b_iso] = {} b_vs_region.sa_nn_vs_region_dict[b_vs_region.b_iso] = {} b_vs_region.sa_ratio_b_vs_region_dict[b_vs_region.b_iso] = {} b_vs_region.b_vs_region_dict[b_vs_region.b_iso][threshold] = nn b_vs_region.sa_nn_vs_region_dict[b_vs_region.b_iso][threshold] = None b_vs_region.sa_ratio_b_vs_region_dict[b_vs_region.b_iso][threshold] = None return overall_score, has_sufficient_regions, \ too_low, too_high, expected_regions, ok def choose_threshold(b_vs_region = None, map_data = None, fraction_occupied = None, solvent_fraction = None, n_residues = None, ncs_copies = None, scale = 0.95, calculate_sa = None, # calculate surface area of top sa_percent of target sa_percent = None, # calculate surface area of top sa_fraction of target density_threshold = None, starting_density_threshold = None, wrapping = None, residues_per_region = None, min_volume = None, min_ratio = None, max_ratio_to_target = None, min_ratio_to_target = None, min_ratio_of_ncs_copy_to_first = None, verbose = None, crystal_symmetry = None, chain_type = None, out = sys.stdout): best_threshold = None best_threshold_has_sufficient_regions = None best_score = None best_ok = None if not ncs_copies: ncs_copies = 1 print("\nChecking possible cutoffs for region identification", file = out) print("Scale: %7.3f" %(scale), file = out) used_ranges = [] # Assume any threshold that is lower than a threshold that gave a non-zero value # and is zero is an upper bound on the best value. Same the other way around upper_bound = 1000 lower_bound = 0.0001 best_nn = None if density_threshold is not None: # use it print("\nUsing input threshold of %5.2f " %( density_threshold), file = out) n_range_low_high_list = [[0, 0]] # use as is else: n_range_low_high_list = [[-16, 4], [-32, 16], [-64, 80]] if starting_density_threshold is not None: starting_density_threshold = starting_density_threshold print("Starting density threshold is: %7.3f" %( starting_density_threshold), file = out) else: starting_density_threshold = 1.0 if verbose: local_out = out else: from libtbx.utils import null_out local_out = null_out() target_in_all_regions = map_data.size()*fraction_occupied*(1-solvent_fraction) print("\nTarget number of points in all regions: %.0f" %( target_in_all_regions), file = local_out) local_threshold = find_threshold_in_map(target_points = int( target_in_all_regions), map_data = map_data) print("Cutoff will be threshold of %7.2f marking %7.1f%% of cell" %( local_threshold, 100.*(1.-solvent_fraction)), file = out) print("B-iso Threshold Target N Biggest Median Ratio Enough Score OK Regions", file = local_out) unique_expected_regions = None for n_range_low, n_range_high in n_range_low_high_list: last_score = None for nn in range(n_range_low, n_range_high+1): if nn in used_ranges: continue used_ranges.append(nn) if density_threshold is not None: threshold = density_threshold else: threshold = starting_density_threshold*(scale**nn) if threshold < lower_bound or threshold > upper_bound: continue co, sorted_by_volume, min_b, max_b = get_co( map_data = map_data.deep_copy(), threshold = threshold, wrapping = wrapping) if len(sorted_by_volume)<2: score, has_sufficient_regions, too_low, too_high, expected_regions, ok = \ None, None, None, None, None, None continue # don't go on else: score, has_sufficient_regions, too_low, too_high, expected_regions, ok = \ score_threshold(b_vs_region = b_vs_region, threshold = threshold, sorted_by_volume = sorted_by_volume, fraction_occupied = fraction_occupied, solvent_fraction = solvent_fraction, residues_per_region = residues_per_region, min_volume = min_volume, min_ratio = min_ratio, max_ratio_to_target = max_ratio_to_target, min_ratio_to_target = min_ratio_to_target, min_ratio_of_ncs_copy_to_first = min_ratio_of_ncs_copy_to_first, ncs_copies = ncs_copies, n_residues = n_residues, map_data = map_data, target_in_all_regions = target_in_all_regions, crystal_symmetry = crystal_symmetry, chain_type = chain_type, out = local_out) if expected_regions: unique_expected_regions = max(1, (ncs_copies-1+expected_regions)//ncs_copies) if too_high and thresholdlower_bound: lower_bound = threshold if score is None: if best_threshold and best_threshold_has_sufficient_regions: if threshold >best_threshold: # new upper bound upper_bound = threshold elif threshold best_score): best_threshold = threshold best_threshold_has_sufficient_regions = has_sufficient_regions best_score = score best_ok = ok if best_threshold is not None: print("\nBest threshold: %5.2f\n" %(best_threshold), file = out) return best_threshold, unique_expected_regions, best_score, best_ok elif density_threshold is not None: # use it anyhow return density_threshold, unique_expected_regions, None, None else: return None, unique_expected_regions, None, None def get_co(map_data = None, threshold = None, wrapping = None): co = maptbx.connectivity(map_data = map_data, threshold = threshold, wrapping = wrapping) regions = co.regions() rr = list(range(0, co.regions().size())) regions_0 = regions[0] rr_0 = rr[0] regions = regions[1:] rr = rr[1:] if rr: z = zip(regions, rr) sorted_by_volume = sorted(z, key=itemgetter(0), reverse = True) else: sorted_by_volume = [] sorted_by_volume = [(regions_0, rr_0)]+sorted_by_volume min_b, max_b = co.get_blobs_boundaries_tuples() # As grid points, not A return co, sorted_by_volume, min_b, max_b def get_connectivity(b_vs_region = None, map_data = None, solvent_fraction = None, n_residues = None, ncs_copies = None, fraction_occupied = None, iterate_with_remainder = None, min_volume = None, min_ratio = None, wrapping = None, residues_per_region = None, max_ratio_to_target = None, min_ratio_to_target = None, min_ratio_of_ncs_copy_to_first = None, starting_density_threshold = None, density_threshold = None, crystal_symmetry = None, chain_type = None, verbose = None, out = sys.stdout): print("\nGetting connectivity", file = out) libtbx.call_back(message = 'segment', data = None) # Normalize map data now to SD of the part that is not solvent map_data = renormalize_map_data( map_data = map_data, solvent_fraction = solvent_fraction) # Try connectivity at various thresholds # Choose one that has about the right number of grid points in top regions scale = 0.95 best_threshold = None best_scale = scale best_score = None best_ok = None best_unique_expected_regions = None for ii in range(3): threshold, unique_expected_regions, score, ok = choose_threshold( density_threshold = density_threshold, starting_density_threshold = starting_density_threshold, b_vs_region = b_vs_region, map_data = map_data, n_residues = n_residues, ncs_copies = ncs_copies, fraction_occupied = fraction_occupied, solvent_fraction = solvent_fraction, scale = scale, wrapping = wrapping, residues_per_region = residues_per_region, min_volume = min_volume, min_ratio = min_ratio, max_ratio_to_target = max_ratio_to_target, min_ratio_to_target = min_ratio_to_target, min_ratio_of_ncs_copy_to_first = min_ratio_of_ncs_copy_to_first, crystal_symmetry = crystal_symmetry, chain_type = chain_type, verbose = verbose, out = out) # Take it if it improves (score, ok) if threshold is not None: if best_score is None or \ ((ok or not best_ok) and (score > best_score)): best_score = score best_unique_expected_regions = unique_expected_regions best_ok = ok best_threshold = threshold best_scale = scale if best_ok or density_threshold is not None: break else: scale = scale**0.333 # keep trying if best_threshold is None or ( density_threshold is not None and best_score is None): if iterate_with_remainder: # on first try failed raise Sorry("No threshold found...try with density_threshold = xxx") else: # on iteration...ok print("Note: No threshold found", file = out) return None, None, None, None, None, None, None, None else: starting_density_threshold = best_threshold # try it next time co, sorted_by_volume, min_b, max_b = get_co( map_data = map_data, threshold = best_threshold, wrapping = wrapping) return co, sorted_by_volume, min_b, max_b, best_unique_expected_regions, \ best_score, threshold, starting_density_threshold def get_volume_of_seq(text, chain_type = None, out = sys.stdout): from iotbx.bioinformatics import chain_type_and_residues # get chain type and residues (or use given chain type and count residues) chain_type, n_residues = chain_type_and_residues(text = text, chain_type = chain_type) if chain_type is None and n_residues is None: return None, None, None if chain_type == 'PROTEIN': mw_residue = 110.0 # from $CDOC/matthews.doc density_factor = 1.23 # 1.66/DENSITY-OF-PROTEIN = 1.66/1.35 else: mw_residue = 330.0 # guess for DNA/RNA density_factor = 1.15 # 1.66/DENSITY-OF-DNA = 1.66/1.45 return len(text)*density_factor*mw_residue, len(text), chain_type def create_rna_dna(cns_dna_rna_residue_names): dd = {} for key in cns_dna_rna_residue_names.keys(): dd[cns_dna_rna_residue_names[key]] = key return dd def get_solvent_content_from_seq_file(params, sequence = None, seq_file = None, overall_chain_type = None, ncs_copies = None, map_volume = None, out = sys.stdout): if params and not overall_chain_type: overall_chain_type = params.crystal_info.chain_type if not sequence and not os.path.isfile(seq_file): raise Sorry( "The sequence file '%s' is missing." %(seq_file)) if not sequence: print("\nReading sequence from %s " %(seq_file), file = out) sequence = open(seq_file).read() from iotbx.bioinformatics import get_sequences sequences = get_sequences(text = sequence) # get unique part of these sequences from mmtbx.validation.chain_comparison import \ extract_unique_part_of_sequences as eups print("Unique part of sequences:", file = out) copies_in_unique, base_copies, unique_sequence_dict = eups(sequences, out = out) all_unique_sequence = [] for seq in copies_in_unique.keys(): print("Copies: %s base copies: %s Sequence: %s" %( copies_in_unique[seq], base_copies, seq), file = out) all_unique_sequence.append(seq) if base_copies != ncs_copies: print("NOTE: %s copies of unique portion but ncs_copies = %s" %( base_copies, ncs_copies), file = out) if ncs_copies == 1: ncs_copies = base_copies print("Using ncs_copies = %s instead" %(ncs_copies), file = out) else: print("Still using ncs_copies = %s" %(ncs_copies), file = out) volume_of_chains = 0. n_residues = 0 chain_types_considered = [] for seq in all_unique_sequence: volume, nres, chain_type = get_volume_of_seq(seq, chain_type = overall_chain_type, out = out) if volume is None: continue volume_of_chains+= volume n_residues+= nres if not chain_type in chain_types_considered: chain_types_considered.append(chain_type) chain_types_considered.sort() print("\nChain types considered: %s\n" %( " ".join(chain_types_considered)), file = out) volume_of_molecules = volume_of_chains*ncs_copies n_residues_times_ncs = n_residues*ncs_copies solvent_fraction = 1.-(volume_of_molecules/map_volume) solvent_fraction = max(0.001, min(0.999, solvent_fraction)) if solvent_fraction == 0.001 or solvent_fraction == 0.999: print("NOTE: solvent fraction of %7.2f very unlikely..." %( solvent_fraction) + "please check ncs_copies and sequence ", file = out) print("Solvent content from composition: %7.2f" %(solvent_fraction), file = out) print("Cell volume: %.1f NCS copies: %d Volume of unique chains: %.1f" %( map_volume, ncs_copies, volume_of_chains), file = out) print("Total residues: %d Volume of all chains: %.1f Solvent fraction: %.3f "%( n_residues_times_ncs, volume_of_molecules, solvent_fraction), file = out) return solvent_fraction, n_residues, n_residues_times_ncs def get_solvent_fraction(params, ncs_object = None, ncs_copies = None, do_not_adjust_dalton_scale = None, sequence = None, seq_file = None, molecular_mass = None, solvent_content = None, crystal_symmetry = None, tracking_data = None, out = sys.stdout): if tracking_data and not crystal_symmetry: #crystal_symmetry = tracking_data.original_crystal_symmetry not used crystal_symmetry = tracking_data.crystal_symmetry map_volume = crystal_symmetry.unit_cell().volume() if tracking_data and not ncs_copies: #ncs_copies = tracking_data.input_ncs_info.original_number_of_operators ncs_copies = tracking_data.input_ncs_info.number_of_operators # 2018-01-29 put back if not ncs_copies: ncs_copies = 1 if params and not solvent_content: solvent_content = params.crystal_info.solvent_content if params and not molecular_mass: molecular_mass = params.crystal_info.molecular_mass if params and not seq_file: seq_file = params.input_files.seq_file if params and not sequence: sequence = params.crystal_info.sequence if seq_file or sequence: solvent_content, n_residues, n_residues_times_ncs = \ get_solvent_content_from_seq_file( params, sequence = sequence, seq_file = seq_file, ncs_copies = ncs_copies, map_volume = map_volume, out = out) if params and not params.crystal_info.solvent_content: params.crystal_info.solvent_content = solvent_content print("Solvent fraction from composition: %7.2f "%( params.crystal_info.solvent_content), file = out) elif params: print("Solvent content from parameters: %7.2f" %( params.crystal_info.solvent_content), file = out) else: if params and params.crystal_info.solvent_content: print("Solvent content from parameters: %7.2f" %( params.crystal_info.solvent_content), file = out) elif molecular_mass: solvent_content = \ get_solvent_fraction_from_molecular_mass( crystal_symmetry = crystal_symmetry, do_not_adjust_dalton_scale = do_not_adjust_dalton_scale, molecular_mass = molecular_mass, out = out) if params: params.crystal_info.solvent_content = solvent_content else: print("Getting solvent content automatically.", file = out) if tracking_data: if params.input_files.seq_file or params.crystal_info.sequence: tracking_data.set_input_seq_info(file_name = params.input_files.seq_file, sequence = params.crystal_info.sequence, n_residues = n_residues) tracking_data.set_n_residues( n_residues = n_residues_times_ncs) if params.crystal_info.solvent_content: tracking_data.set_solvent_fraction(params.crystal_info.solvent_content) return tracking_data else: return solvent_content def top_key(dd): if not dd: return None, None elif len(dd) == 1: return list(dd.items())[0] else: best_key = None best_n = None for key in dd.keys(): if not best_n or dd[key] > best_n: best_n = dd[key] best_key = key return best_key, best_n def choose_max_regions_to_consider(params, sorted_by_volume = None, ncs_copies = None): max_per_au = params.segmentation.max_per_au min_ratio = params.segmentation.min_ratio min_volume = params.segmentation.min_volume # sort and eliminate regions with few points and those at end of list if len(sorted_by_volume)<2: return 0 max_grid_points = sorted_by_volume[1][0] cntr = 0 for p in sorted_by_volume[1:]: cntr+= 1 if max_per_au and (cntr>max_per_au*ncs_copies): cntr-= 1 break v, i = p # v = volume in grid points, i = id if v/max_grid_points1: # Skip if no ncs... for id in region_scattered_points_dict.keys(): for xyz_cart in region_scattered_points_dict[id]: n = 0 for i0 in range(len(ncs_group.translations_orth())): if i0 == identity_op: continue r = ncs_group.rota_matrices_inv()[i0] # inverse maps pos 0 on to pos i t = ncs_group.translations_orth_inv()[i0] n+= 1 new_xyz_cart = r * matrix.col(xyz_cart) + t new_xyz_frac = unit_cell.fractionalize(new_xyz_cart) if tracking_data.params.crystal_info.use_sg_symmetry or \ (new_xyz_frac[0]>= 0 and new_xyz_frac[0]<= 1 and \ new_xyz_frac[1]>= 0 and new_xyz_frac[1]<= 1 and \ new_xyz_frac[2]>= 0 and new_xyz_frac[2]<= 1): value = edited_mask.value_at_closest_grid_point(new_xyz_frac) else: value = 0 # value for nothing there 2017-12-16 if value == id: duplicate_dict[id]+= 1 break # only count once elif value>0: # notice which one is matched if not value in equiv_dict[id]: equiv_dict[id][value] = 0 equiv_dict_ncs_copy_dict[id][value] = {} equiv_dict[id][value]+= 1 if not n in equiv_dict_ncs_copy_dict[id][value]: equiv_dict_ncs_copy_dict[id][value][n] = 0 equiv_dict_ncs_copy_dict[id][value][n]+= 1 # how many are ncs copy n return duplicate_dict, equiv_dict, equiv_dict_ncs_copy_dict, \ region_range_dict, region_centroid_dict, region_scattered_points_dict def get_region_scattered_points_dict( edited_volume_list = None, edited_mask = None, unit_cell = None, sampling_rate = None, target_points_per_region = None, minimum_points_per_region = None, maximum_points_per_region = None): # Get sampled points in each region sample_dict = {} region_scattered_points_dict = {} # some points in each region if not sampling_rate: sampling_rate = edited_volume_list[0]//target_points_per_region sampling_rate_set = False else: sampling_rate_set = True volumes = flex.int() sampling_rates = flex.int() id_list = [] # have to set up dummy first set: volumes.append(0) sampling_rates.append(0) id_list.append(0) for i in range(len(edited_volume_list)): id = i+1 v = edited_volume_list[i] region_scattered_points_dict[id] = flex.vec3_double() volumes.append(v) if sampling_rate_set: sample_dict[id] = sampling_rate sampling_rates.append(sampling_rate) else: sample_dict[id] = max(1, max(v//maximum_points_per_region, min(v//minimum_points_per_region, sampling_rate) )) sampling_rates.append(max(1, max(v//maximum_points_per_region, min(v//minimum_points_per_region, sampling_rate) ))) id_list.append(id) sample_regs_obj = maptbx.sample_all_mask_regions( mask = edited_mask, volumes = volumes, sampling_rates = sampling_rates, unit_cell = unit_cell) for id in id_list[1:]: # skip the dummy first set region_scattered_points_dict[id] = sample_regs_obj.get_array(id) return region_scattered_points_dict def remove_bad_regions(params = None, duplicate_dict = None, edited_volume_list = None, out = sys.stdout): worst_list = [] for id in list(duplicate_dict.keys()): fract = duplicate_dict[id]/edited_volume_list[id-1] if duplicate_dict[id] and fract >= params.segmentation.max_overlap_fraction: worst_list.append([fract, id]) else: del duplicate_dict[id] worst_list.sort() worst_list.reverse() bad_region_list = [] max_number_to_remove = int(0.5+ 0.01*params.segmentation.remove_bad_regions_percent*len(edited_volume_list)) if worst_list: print("\nRegions that span multiple NCS au:", file = out) for fract, id in worst_list: print("ID: %d Duplicate points: %d (%.1f %%)" %( id, duplicate_dict[id], 100.*fract), end = ' ', file = out) if len(bad_region_list)= ncs_copies-1: break return equiv_dict_ncs_copy def get_overlap(l1, l2): overlap_list = [] l1a = single_list(l1) l2a = single_list(l2) for i in l1a: if i in l2a and not i in overlap_list: overlap_list.append(i) return overlap_list def group_ncs_equivalents(params, region_list = None, region_volume_dict = None, equiv_dict_ncs_copy = None, tracking_data = None, split_if_possible = None, out = sys.stdout): # equiv_dict_ncs_copy[id][id1] = ncs_copy # group together all the regions that are related to region 1...etc # if split_if_possible then skip all groups with multiple entries ncs_equiv_groups_as_list = [] ncs_equiv_groups_as_dict = {} for id in region_list: equiv_group = {} #equiv_group[ncs_copy] = [id1, id2, id3...] equiv_group[0] = [id] # always for id1 in equiv_dict_ncs_copy.get(id, {}).keys(): ncs_copy = equiv_dict_ncs_copy[id][id1] if not ncs_copy in equiv_group: equiv_group[ncs_copy] = [] equiv_group[ncs_copy].append(id1) # id1 is ncs_copy of id all_single = True equiv_group_as_list = [] total_grid_points = 0 missing_ncs_copies = [] present_ncs_copies = [] for ncs_copy in range(tracking_data.input_ncs_info.number_of_operators): # goes 0 to ncs_copies-1 (including extra ones if present) local_equiv_group = equiv_group.get(ncs_copy, []) if local_equiv_group: equiv_group_as_list.append(local_equiv_group) present_ncs_copies.append(ncs_copy) if ncs_copy > 0 and \ len(local_equiv_group)>1 and len(equiv_group.get(0, [])) == 1: all_single = False for id in equiv_group.get(ncs_copy, []): total_grid_points+= region_volume_dict[id] else: missing_ncs_copies.append(ncs_copy) equiv_group_as_list.sort() if tracking_data.input_ncs_info.is_helical_symmetry: # complete if we have original_number_of_operators worth if (not params.segmentation.require_complete) or \ len(present_ncs_copies)>= \ tracking_data.input_ncs_info.original_number_of_operators: complete = True else: complete = False else: if len(missing_ncs_copies) == 0: complete = True else: complete = False if complete and \ (not str(equiv_group_as_list) in ncs_equiv_groups_as_dict or total_grid_points>ncs_equiv_groups_as_dict[str(equiv_group_as_list)]) \ and (all_single or (not split_if_possible)): ncs_equiv_groups_as_dict[str(equiv_group_as_list)] = total_grid_points ncs_equiv_groups_as_list.append([total_grid_points, equiv_group_as_list]) ncs_equiv_groups_as_list.sort() ncs_equiv_groups_as_list.reverse() # Now remove any group that duplicates a previous group # 2015-11-07 allow a member to be in multiple groups though (for example # one that spans several groups because it contains 2 region in other ncs # copies) # Make sure that if there are duplicates they are all in the leading # positions of the list (these must be very big ones as they match 2 # regions in other ncs copies) max_duplicates = tracking_data.input_ncs_info.number_of_operators-1 # not all duplicates ncs_group_list = [] used_list = [] print("All equiv groups:", file = out) used_regions = [] for total_grid_points, equiv_group_as_list in ncs_equiv_groups_as_list: duplicate = False n_dup = 0 for equiv_group in equiv_group_as_list: for x in equiv_group: if x in used_list: n_dup+= 1 if n_dup>max_duplicates or n_dup >len(equiv_group_as_list)-1: duplicate = True if not duplicate and n_dup>0: # check carefully to make sure that all # are leading entries for ncs_group in ncs_group_list: overlaps = get_overlap(ncs_group, equiv_group_as_list) if not overlaps: continue overlaps.sort() expected_match = single_list(equiv_group_as_list)[:len(overlaps)] expected_match.sort() if overlaps!= expected_match: # not leading entries duplicate = True break if not duplicate: #print >>out, "NCS GROUP:", equiv_group_as_list, ":", total_grid_points ncs_group_list.append(equiv_group_as_list) for equiv_group in equiv_group_as_list: for x in equiv_group: if not x in used_list: used_list.append(x) print("Total NCS groups: %d" %len(ncs_group_list), file = out) # Make a dict that lists all ids that are in the same group as region x shared_group_dict = {} for ncs_group in ncs_group_list: for group_list in ncs_group: for id1 in group_list: if not id1 in shared_group_dict: shared_group_dict[id1] = [] for other_group_list in ncs_group: if other_group_list is group_list:continue for other_id1 in other_group_list: if not other_id1 in shared_group_dict [id1]: shared_group_dict[id1].append(other_id1) return ncs_group_list, shared_group_dict def identify_ncs_regions(params, sorted_by_volume = None, co = None, min_b = None, max_b = None, ncs_obj = None, tracking_data = None, out = sys.stdout): # 1.choose top regions to work with # 2.remove regions that are in more than one au of the NCS # 3.identify groups of regions that are related by NCS # Also note the centers and bounds of each region # Choose number of top regions to consider max_regions_to_consider = choose_max_regions_to_consider(params, sorted_by_volume = sorted_by_volume, ncs_copies = tracking_data.input_ncs_info.original_number_of_operators) print("\nIdentifying NCS-related regions.Total regions to consider: %d" %( max_regions_to_consider), file = out) if max_regions_to_consider<1: print("\nUnable to identify any NCS regions", file = out) return None, tracking_data, None # Go through all grid points; discard if not in top regions # Renumber regions in order of decreasing size load_saved_files = False # set to True to load results from previous run dump_files = False # set to True to dump results and speed up next run if not load_saved_files: edited_mask, edited_volume_list, original_id_from_id = get_edited_mask( sorted_by_volume = sorted_by_volume, co = co, max_regions_to_consider = max_regions_to_consider, out = out) if dump_files: from libtbx import easy_pickle easy_pickle.dump("edited_mask.pkl", [edited_mask, edited_volume_list, original_id_from_id]) else: from libtbx import easy_pickle [edited_mask, edited_volume_list, original_id_from_id ] = easy_pickle.load("edited_mask.pkl") print("Loading edited_mask.pkl", file = out) # edited_mask contains re-numbered region id's # Identify duplicate and ncs relationships between regions # duplicate_dict[id] = number of duplicates for that region # equiv_dict[id][other_id] = number_of points other_id matches # id through an ncs relationship if not load_saved_files: duplicate_dict, equiv_dict, equiv_dict_ncs_copy_dict, \ region_range_dict, region_centroid_dict, \ region_scattered_points_dict = \ run_get_duplicates_and_ncs( ncs_obj = ncs_obj, min_b = min_b, max_b = max_b, edited_mask = edited_mask, original_id_from_id = original_id_from_id, edited_volume_list = edited_volume_list, max_regions_to_consider = max_regions_to_consider, tracking_data = tracking_data, out = out) # Remove any bad regions region_list, region_volume_dict, new_sorted_by_volume, \ bad_region_list = remove_bad_regions( params = params, duplicate_dict = duplicate_dict, edited_volume_list = edited_volume_list, out = out) # Identify groups of regions that are ncs-related # equiv_dict_ncs_copy[id][id1] = ncs_copy of id that corresponds to id1 equiv_dict_ncs_copy = get_ncs_equivalents( region_list = region_list, bad_region_list = bad_region_list, region_scattered_points_dict = region_scattered_points_dict, equiv_dict = equiv_dict, ncs_copies = tracking_data.input_ncs_info.number_of_operators, equiv_dict_ncs_copy_dict = equiv_dict_ncs_copy_dict, out = out) if dump_files: from libtbx import easy_pickle easy_pickle.dump("save.pkl", [duplicate_dict, equiv_dict, region_range_dict, region_centroid_dict, region_scattered_points_dict, region_list, region_volume_dict, new_sorted_by_volume, bad_region_list, equiv_dict_ncs_copy, tracking_data]) print("Dumped save.pkl", file = out) else: from libtbx import easy_pickle [duplicate_dict, equiv_dict, region_range_dict, region_centroid_dict, region_scattered_points_dict, region_list, region_volume_dict, new_sorted_by_volume, bad_region_list, equiv_dict_ncs_copy, tracking_data] = easy_pickle.load("save.pkl") print("Loaded save.pkl", file = out) # Group together regions that are ncs-related. Also if one ncs # copy has 2 or more regions linked together, group the other ones. # each entry in ncs_group_list is a list of regions for each ncs_copy: # e.g., [[8], [9, 23], [10, 25], [11, 27], [12, 24], [13, 22], [14, 26]] # May contain elements that are in bad_region_list (to exclude later) if not load_saved_files: ncs_group_list, shared_group_dict = group_ncs_equivalents(params, split_if_possible = params.segmentation.split_if_possible, tracking_data = tracking_data, region_volume_dict = region_volume_dict, region_list = region_list, equiv_dict_ncs_copy = equiv_dict_ncs_copy, out = out) if dump_files: from libtbx import easy_pickle easy_pickle.dump("group_list.pkl", [ncs_group_list, shared_group_dict]) print("Dumped to group_list.pkl", file = out) else: from libtbx import easy_pickle [ncs_group_list, shared_group_dict] = easy_pickle.load("group_list.pkl") print("Loaded group_list.pkl", file = out) ncs_group_obj = ncs_group_object( ncs_group_list = ncs_group_list, shared_group_dict = shared_group_dict, ncs_obj = ncs_obj, crystal_symmetry = tracking_data.crystal_symmetry, edited_mask = edited_mask, origin_shift = tracking_data.origin_shift, co = co, min_b = min_b, max_b = max_b, equiv_dict = equiv_dict, bad_region_list = bad_region_list, original_id_from_id = original_id_from_id, edited_volume_list = edited_volume_list, region_range_dict = region_range_dict, region_scattered_points_dict = region_scattered_points_dict, region_centroid_dict = region_centroid_dict) return ncs_group_obj, tracking_data, equiv_dict_ncs_copy def get_center_list(regions, region_centroid_dict = None): center_list = [] for region in regions: center_list.append(region_centroid_dict[region]) return center_list def get_average_center(regions, region_centroid_dict = None): center_list = get_center_list(regions, region_centroid_dict = region_centroid_dict) for region in regions: center_list.append(region_centroid_dict[region]) average_center = deepcopy(center_list[0]) if len(center_list)>1: for r in center_list[1:]: for i in range(3): average_center[i]+= r[i] for i in range(3): average_center[i]/= len(center_list) return average_center def get_dist(r, s): dd = 0. for i in range(3): dd+= (r[i]-s[i])**2 return dd**0.5 def has_intersection(set1, set2): set1a = single_list(set1) set2a = single_list(set2) for x in set1a: if x in set2a: return True return False def get_scattered_points_list(other_regions, region_scattered_points_dict = None): scattered_points_list = flex.vec3_double() for x in other_regions: scattered_points_list.extend(region_scattered_points_dict[x]) return scattered_points_list def get_inter_region_dist_dict(ncs_group_obj = None, selected_regions = None, target_scattered_points = None): dd = {} for i in range(len(selected_regions)): id = selected_regions[i] if not id in dd: dd[id] = {} test_centers = ncs_group_obj.region_scattered_points_dict[id] for j in range(i+1, len(selected_regions)): id1 = selected_regions[j] test_centers1 = ncs_group_obj.region_scattered_points_dict[id1] dist = get_closest_dist(test_centers, test_centers1) dd[id][id1] = dist if not id1 in dd: dd[id1] = {} dd[id1][id] = dist return dd def get_dist_to_first_dict(ncs_group_obj = None, selected_regions = None, inter_region_dist_dict = None, target_scattered_points = None): # Get distance to region 0 ( or to target_scattered_points if supplied) dist_to_first_dict = {} if target_scattered_points: start_region = 0 for x in selected_regions: dist_to_first_dict[x] = get_closest_dist( ncs_group_obj.region_scattered_points_dict[x], target_scattered_points) else: start_region = 1 x0 = selected_regions[0] dist_to_first_dict[x0] = 0 for x in selected_regions[1:]: dist_to_first_dict[x] = inter_region_dist_dict[x0][x] changing = True while changing: changing = False for x in selected_regions[start_region:]: for y in selected_regions[start_region:]: if x == y: continue if dist_to_first_dict[y]1: rms/= rms_n rms = rms**0.5 return rms def get_rms(selected_regions = None, region_centroid_dict = None): # return rms distance of each region center from average of all others rms = 0. rms_n = 0. for x in selected_regions: other_regions = remove_one_item(selected_regions, item_to_remove = x) current_center = get_average_center(other_regions, region_centroid_dict = region_centroid_dict) test_center = region_centroid_dict[x] dist = get_dist(current_center, test_center) rms+= dist**2 rms_n+= 1. if rms_n>1: rms/= rms_n return rms**0.5 def single_list(list_of_lists): single = [] for x in list_of_lists: if type(x) == type([1, 2, 3]): single+= single_list(x) else: single.append(x) return single def get_closest_dist(test_center, target_centers): # make sure we have target_centers = vec3_double and not a list, # and vec3_double or tuple for test_center if type(test_center) == type([1, 2, 3]): test_center = flex.vec3_double(test_center) if type(target_centers) == type([1, 2, 3]): target_centers = flex.vec3_double(target_centers) if test_center.size()<1 or target_centers.size()<1: return None closest_dist = test_center.min_distance_between_any_pair(target_centers) return closest_dist def region_lists_have_ncs_overlap(set1, set2, ncs_group_obj = None, cutoff = 0): for id1 in set1: for id2 in set2: if id2 in ncs_group_obj.shared_group_dict.get(id1, []): return True return False def get_effective_radius(ncs_group_obj = None, target_scattered_points = None, weight_rad_gyr = None, selected_regions = None): sr = deepcopy(selected_regions) sr.sort() rad_gyr = get_radius_of_gyration(ncs_group_obj = ncs_group_obj, selected_regions = sr) rms = get_closest_neighbor_rms(ncs_group_obj = ncs_group_obj, target_scattered_points = target_scattered_points, selected_regions = sr) max_cell_dim = 0. if ncs_group_obj.max_cell_dim and ncs_group_obj.max_cell_dim > 1.0: wrg = weight_rad_gyr*(300/ncs_group_obj.max_cell_dim) # have a consistent scale else: wrg = weight_rad_gyr effective_radius = (rms+wrg*rad_gyr)/(1.+wrg) return effective_radius def add_neighbors(params, selected_regions = None, max_length_of_group = None, target_scattered_points = None, tracking_data = None, equiv_dict_ncs_copy = None, ncs_group_obj = None, out = sys.stdout): # Add neighboring regions on to selected_regions. # Same rules as select_from_seed selected_regions = single_list(deepcopy(selected_regions)) added_regions = [] start_dist = get_effective_radius(ncs_group_obj = ncs_group_obj, target_scattered_points = target_scattered_points, weight_rad_gyr = params.segmentation.weight_rad_gyr, selected_regions = selected_regions) delta_dist = params.segmentation.add_neighbors_dist max_dist = start_dist+delta_dist starting_selected_regions = deepcopy(selected_regions) for x in selected_regions: # delete, add in alternatives one at a time and # keep all the ok ones ncs_groups_to_use = get_ncs_related_regions( ncs_group_obj = ncs_group_obj, selected_regions = [x], include_self = False) for x in ncs_groups_to_use: # try adding from each group if x in selected_regions+added_regions: continue ncs_group = [[x]] current_scattered_points_list = get_scattered_points_list(selected_regions, region_scattered_points_dict = ncs_group_obj.region_scattered_points_dict) for ncs_set in ncs_group: # pick the best ncs_set from this group if has_intersection(ncs_group_obj.bad_region_list, ncs_set): continue dist = get_effective_radius(ncs_group_obj = ncs_group_obj, target_scattered_points = target_scattered_points, weight_rad_gyr = params.segmentation.weight_rad_gyr, selected_regions = selected_regions+ncs_set) if dist <= max_dist: added_regions.append(x) selected_regions = selected_regions+added_regions dist = get_effective_radius(ncs_group_obj = ncs_group_obj, target_scattered_points = target_scattered_points, weight_rad_gyr = params.segmentation.weight_rad_gyr, selected_regions = selected_regions) # Identify all the NCS operators required to map final to starting # equiv_dict_ncs_copy[id][id1] = ncs_copy of id that corresponds to id1 ncs_group = ncs_group_obj.ncs_obj.ncs_groups()[0] identity_op = ncs_group.identity_op_id() ncs_ops_used = [identity_op] for id in selected_regions: related_regions = get_ncs_related_regions( ncs_group_obj = ncs_group_obj, selected_regions = [id], include_self = False) for id1 in selected_regions: if not id1 in related_regions: continue ncs_copy1 = equiv_dict_ncs_copy.get(id, {}).get(id1, None) ncs_copy2 = equiv_dict_ncs_copy.get(id1, {}).get(id, None) for a in [ncs_copy1, ncs_copy2]: if a is not None and not a in ncs_ops_used: ncs_ops_used.append(a) selected_regions.sort() ncs_ops_used.sort() for x in selected_regions: print("GROUP ", x, ":", ncs_group_obj.shared_group_dict.get(x, []), file = out) return selected_regions, dist, ncs_ops_used def select_from_seed(params, starting_regions, target_scattered_points = None, max_length_of_group = None, ncs_groups_to_use = None, tracking_data = None, ncs_group_obj = None): selected_regions = single_list(deepcopy(starting_regions)) # do not allow any region in ncs_group_obj.bad_region_list # also do not allow any region that is in an ncs-related group to any region # already used. Use ncs_group_obj.equiv_dict to identify these. if not ncs_groups_to_use: ncs_groups_to_use = ncs_group_obj.ncs_group_list for ncs_group in ncs_groups_to_use: # try adding from each group if max_length_of_group is not None and \ len(selected_regions)>= max_length_of_group: break best_ncs_set = None best_dist = None if has_intersection(ncs_group, selected_regions): continue current_scattered_points_list = get_scattered_points_list(selected_regions, region_scattered_points_dict = ncs_group_obj.region_scattered_points_dict) if target_scattered_points: current_scattered_points_list.extend(target_scattered_points) for ncs_set in ncs_group: # pick the best ncs_set from this group if has_intersection(ncs_group_obj.bad_region_list, ncs_set): continue # does any ncs copy of anything in selected_regions actually overlap # with any member of ncs_set... might be efficient to delete the entire # ncs_group if any ncs_set overlaps, but could lose some. if region_lists_have_ncs_overlap(ncs_set, selected_regions, ncs_group_obj = ncs_group_obj): continue dist = get_effective_radius(ncs_group_obj = ncs_group_obj, target_scattered_points = target_scattered_points, weight_rad_gyr = params.segmentation.weight_rad_gyr, selected_regions = selected_regions+ncs_set) if best_dist is None or dist= params.segmentation.seeds_to_try: break # don't bother to keep trying if starting_region and starting_region in ncs_group_obj.bad_region_list: continue # do not use if starting_region: # NOTE: starting_region is a list itself starting_region_list = [starting_region] else: starting_region_list = [] selected_regions, rms = select_from_seed(params, starting_region_list, target_scattered_points = target_scattered_points, max_length_of_group = max_length_of_group, tracking_data = tracking_data, ncs_group_obj = ncs_group_obj) if not selected_regions: continue ok_seeds_examined+= 1 if best_rms is None or rms1: print("\nAdding neighbor groups...", file = out) selected_regions, rms, ncs_ops_used = add_neighbors(params, selected_regions = selected_regions, max_length_of_group = max_length_of_group, target_scattered_points = target_scattered_points, equiv_dict_ncs_copy = equiv_dict_ncs_copy, tracking_data = tracking_data, ncs_group_obj = ncs_group_obj, out = out) else: ncs_ops_used = None print("\nFinal selected regions with rms of %6.2f: " %(rms), end = ' ', file = out) for x in selected_regions: print(x, end = ' ', file = out) if ncs_ops_used: print("\nNCS operators used: ", end = ' ', file = out) for op in ncs_ops_used: print(op, end = ' ', file = out) print(file = out) # Save an ncs object containing just the ncs_ops_used ncs_group_obj.set_ncs_ops_used(ncs_ops_used) # Identify scattered points for all selected regions: scattered_points = get_scattered_points_list(selected_regions, region_scattered_points_dict = ncs_group_obj.region_scattered_points_dict) # Identify ncs-related regions for all the selected regions self_and_ncs_related_regions = get_ncs_related_regions( ncs_group_obj = ncs_group_obj, selected_regions = selected_regions, include_self = True) ncs_related_regions = get_ncs_related_regions( ncs_group_obj = ncs_group_obj, selected_regions = selected_regions, include_self = False) print("NCS-related regions (not used): %d " %(len(ncs_related_regions)), file = out) ncs_group_obj.set_selected_regions(selected_regions) ncs_group_obj.set_self_and_ncs_related_regions(self_and_ncs_related_regions) ncs_group_obj.set_ncs_related_regions(ncs_related_regions) return ncs_group_obj, scattered_points def get_bool_mask_as_int(ncs_group_obj = None, mask_as_int = None, mask_as_bool = None): if mask_as_int: mask_as_int = mask_as_int.deep_copy() else: mask_as_int = ncs_group_obj.edited_mask.deep_copy() s = (mask_as_bool == True) mask_as_int = mask_as_int.set_selected(s, 1) mask_as_int = mask_as_int.set_selected(~s, 0) return mask_as_int def get_bool_mask_of_regions(ncs_group_obj = None, region_list = None, expand_size = None): s = (ncs_group_obj.edited_mask == -1) if region_list is None: region_list = [] for id in region_list: if not expand_size: s |= (ncs_group_obj.edited_mask == id) # just take this region else: # expand the size of the regions...use expand_mask which operates # on the original id numbers and uses the co bool_region_mask = ncs_group_obj.co.expand_mask( id_to_expand = ncs_group_obj.original_id_from_id[id], expand_size = expand_size) s |= (bool_region_mask == True) bool_mask = ncs_group_obj.co.expand_mask(id_to_expand = 1, expand_size = 1) # just to get bool mask bool_mask = bool_mask.set_selected(s, True) bool_mask = bool_mask.set_selected(~s, False) return bool_mask def create_remaining_mask_and_map(params, ncs_group_obj = None, map_data = None, crystal_symmetry = None, out = sys.stdout): if not ncs_group_obj.selected_regions: print("No regions selected", file = out) return map_data # create new remaining_map containing everything except the part that # has been interpreted (and all points in interpreted NCS-related copies) bool_all_used = get_bool_mask_of_regions(ncs_group_obj = ncs_group_obj, region_list = ncs_group_obj.selected_regions+ ncs_group_obj.self_and_ncs_related_regions, expand_size = params.segmentation.expand_size) map_data_remaining = map_data.deep_copy() s = (bool_all_used == True) map_data_remaining = map_data_remaining.set_selected(s, params.segmentation.value_outside_mask) return map_data_remaining def get_lower(lower_bounds, lower): new_lower = [] for i in range(3): if lower_bounds[i] is None: new_lower.append(lower[i]) elif lower[i] is None: new_lower.append(lower_bounds[i]) else: new_lower.append(min(lower_bounds[i], lower[i])) return new_lower def get_upper(upper_bounds, upper): new_upper = [] for i in range(3): if upper_bounds[i] is None: new_upper.append(upper[i]) elif upper[i] is None: new_upper.append(upper_bounds[i]) else: new_upper.append(max(upper_bounds[i], upper[i])) return new_upper def get_bounds(ncs_group_obj = None, id = None): orig_id = ncs_group_obj.original_id_from_id[id] lower = ncs_group_obj.min_b[orig_id] upper = ncs_group_obj.max_b[orig_id] return lower, upper def get_selected_and_related_regions(params, ncs_group_obj = None): # Identify all points in the targeted regions bool_selected_regions = get_bool_mask_of_regions(ncs_group_obj = ncs_group_obj, region_list = ncs_group_obj.selected_regions, expand_size = params.segmentation.expand_size+\ params.segmentation.mask_additional_expand_size) # and all points in NCS-related copies (to be excluded) if params.segmentation.exclude_points_in_ncs_copies and ( not params.segmentation.add_neighbors): bool_ncs_related_mask = get_bool_mask_of_regions(ncs_group_obj = ncs_group_obj, region_list = ncs_group_obj.ncs_related_regions) # NOTE: using ncs_related_regions here NOT self_and_ncs_related_regions else: bool_ncs_related_mask = None lower_bounds = [None, None, None] upper_bounds = [None, None, None] if ncs_group_obj.selected_regions: for id in ncs_group_obj.selected_regions: lower, upper = get_bounds( ncs_group_obj = ncs_group_obj, id = id) lower_bounds = get_lower(lower_bounds, lower) upper_bounds = get_upper(upper_bounds, upper) return bool_selected_regions, bool_ncs_related_mask, lower_bounds, upper_bounds def adjust_bounds(params, lower_bounds, upper_bounds, map_data = None, out = sys.stdout): # range is lower_bounds to upper_bounds lower_bounds = list(lower_bounds) upper_bounds = list(upper_bounds) if params is None or params.output_files.box_buffer is None: box_buffer = 0 else: box_buffer = int(0.5+params.output_files.box_buffer) for i in range(3): if lower_bounds[i] is None: lower_bounds[i] = 0 if upper_bounds[i] is None: upper_bounds[i] = 0 lower_bounds[i]-= box_buffer lower_bounds[i] = max(0, lower_bounds[i]) upper_bounds[i]+= box_buffer upper_bounds[i] = min(map_data.all()[i]-1, upper_bounds[i]) """ print >>out, "\nRange: X:(%6d, %6d) Y:(%6d, %6d) Z:(%6d, %6d)" %( lower_bounds[0], upper_bounds[0], lower_bounds[1], upper_bounds[1], lower_bounds[2], upper_bounds[2]) """ return lower_bounds, upper_bounds def write_region_maps(params, ncs_group_obj = None, map_data = None, tracking_data = None, remainder_ncs_group_obj = None, regions_to_skip = None, out = sys.stdout): remainder_regions_written = [] map_files_written = [] if not ncs_group_obj: return map_files_written, remainder_regions_written if not ncs_group_obj.selected_regions: return map_files_written, remainder_regions_written for id in ncs_group_obj.selected_regions: if regions_to_skip and id in regions_to_skip: print("Skipping remainder region %d (already written out)" %(id), file = out) continue print("Writing region %d" %(id), end = ' ', file = out) # dummy atoms representing this region sites = ncs_group_obj.region_scattered_points_dict[id] bool_region_mask = ncs_group_obj.co.expand_mask( id_to_expand = ncs_group_obj.original_id_from_id[id], expand_size = params.segmentation.expand_size) s = (bool_region_mask == True) lower_bounds, upper_bounds = get_bounds(ncs_group_obj = ncs_group_obj, id = id) if remainder_ncs_group_obj: for remainder_id in remainder_ncs_group_obj.remainder_id_dict.keys(): if remainder_ncs_group_obj.remainder_id_dict[remainder_id] == id: remainder_regions_written.append(remainder_id) sites.extend( remainder_ncs_group_obj.region_scattered_points_dict[remainder_id]) print("(including remainder region %d)" %(remainder_id), end = ' ', file = out) remainder_bool_region_mask = remainder_ncs_group_obj.co.expand_mask( id_to_expand = remainder_ncs_group_obj.original_id_from_id[remainder_id], expand_size = params.segmentation.expand_size) s|= (remainder_bool_region_mask == True) lower, upper = get_bounds( ncs_group_obj = remainder_ncs_group_obj, id = remainder_id) lower_bounds = get_lower(lower_bounds, lower) upper_bounds = get_upper(upper_bounds, upper) region_mask = map_data.deep_copy() region_mask = region_mask.set_selected(s, 1) region_mask = region_mask.set_selected(~s, 0) local_map_data = map_data.deep_copy() local_map_data = local_map_data * region_mask.as_double() # Now cut down the map to the size we want lower_bounds, upper_bounds = adjust_bounds(params, lower_bounds, upper_bounds, map_data = map_data, out = out) box_map, box_crystal_symmetry, \ dummy_smoothed_box_mask_data, dummy_original_box_map_data = cut_out_map( map_data = local_map_data, \ crystal_symmetry = tracking_data.crystal_symmetry, min_point = lower_bounds, max_point = upper_bounds, out = out) if remainder_ncs_group_obj: text = "" else: text = "_r" base_file = 'map%s_%d.ccp4' %(text, id) base_pdb_file = 'atoms%s_%d.pdb' %(text, id) if tracking_data.params.output_files.output_directory: if not os.path.isdir(tracking_data.params.output_files.output_directory): os.mkdir(tracking_data.params.output_files.output_directory) file_name = os.path.join(tracking_data.params.output_files.output_directory, base_file) pdb_file_name = os.path.join( tracking_data.params.output_files.output_directory, base_pdb_file) else: file_name = base_file pdb_file_name = base_pdb_file write_ccp4_map(box_crystal_symmetry, file_name, box_map) print("to %s" %(file_name), file = out) map_files_written.append(file_name) tracking_data.add_output_region_map_info( file_name = file_name, crystal_symmetry = box_crystal_symmetry, origin = box_map.origin(), all = box_map.all(), map_id = base_file) print("Atoms representation written to %s" %(pdb_file_name), file = out) write_atoms(tracking_data = tracking_data, sites = sites, file_name = pdb_file_name, out = out) tracking_data.add_output_region_pdb_info( file_name = pdb_file_name) return map_files_written, remainder_regions_written def get_bounds_from_sites(sites_cart = None, map_data = None, unit_cell = None): lower_bounds = [None, None, None] upper_bounds = [None, None, None] sites_frac = unit_cell.fractionalize(sites_cart) nx, ny, nz = map_data.all() for x_frac in sites_frac: x = [ int(0.5+nx*x_frac[0]), int(0.5+ny*x_frac[1]), int(0.5+nz*x_frac[2])] if lower_bounds[0] is None or x[0]upper_bounds[0]: upper_bounds[0] = x[0] if upper_bounds[1] is None or x[1]>upper_bounds[1]: upper_bounds[1] = x[1] if upper_bounds[2] is None or x[2]>upper_bounds[2]: upper_bounds[2] = x[2] return lower_bounds, upper_bounds def write_output_files(params, tracking_data = None, map_data = None, half_map_data_list = None, ncs_group_obj = None, remainder_ncs_group_obj = None, pdb_hierarchy = None, removed_ncs = None, out = sys.stdout): half_map_data_list_au = [] if not half_map_data_list: half_map_data_list = [] if params.output_files.au_output_file_stem: au_mask_output_file = os.path.join(tracking_data.params.output_files.output_directory, params.output_files.au_output_file_stem+"_mask.ccp4") au_map_output_file = os.path.join(tracking_data.params.output_files.output_directory, params.output_files.au_output_file_stem+"_map.ccp4") au_atom_output_file = os.path.join(tracking_data.params.output_files.output_directory, params.output_files.au_output_file_stem+"_atoms.pdb") else: au_mask_output_file = None au_map_output_file = None au_atom_output_file = None # Write out pdb file with dummy atoms for the AU to au_atom_output_file if au_atom_output_file and params.output_files.write_output_maps: sites = flex.vec3_double() for id in ncs_group_obj.selected_regions: sites.extend(ncs_group_obj.region_scattered_points_dict[id]) if remainder_ncs_group_obj: for id in remainder_ncs_group_obj.selected_regions: sites.extend(remainder_ncs_group_obj.region_scattered_points_dict[id]) write_atoms(tracking_data = tracking_data, sites = sites, file_name = au_atom_output_file, out = out) tracking_data.set_output_ncs_au_pdb_info(file_name = au_atom_output_file) # Write out mask and map representing one NCS copy and none of # other NCS copies. Expand the mask to include neighboring points (but # not those explicitly in other NCS copies if params.map_modification.soft_mask and params.control.save_box_map_ncs_au: mask_expand_size = estimate_expand_size( crystal_symmetry = tracking_data.crystal_symmetry, map_data = map_data, expand_target = tracking_data.params.segmentation.mask_expand_ratio*\ tracking_data.params.crystal_info.resolution, out = out) params.segmentation.mask_additional_expand_size = max(mask_expand_size, params.segmentation.mask_additional_expand_size, ) bool_selected_regions, bool_ncs_related_mask, lower_bounds, upper_bounds = \ get_selected_and_related_regions( params, ncs_group_obj = ncs_group_obj) if bool_ncs_related_mask is not None: s_ncs_related = (bool_ncs_related_mask == True) else: s_ncs_related = None # Add in remainder regions if present if remainder_ncs_group_obj: bool_remainder_selected_regions, bool_remainder_ncs_related_mask, \ remainder_lower_bounds, remainder_upper_bounds = \ get_selected_and_related_regions( params, ncs_group_obj = remainder_ncs_group_obj) lower_bounds = get_lower(lower_bounds, remainder_lower_bounds) upper_bounds = get_upper(upper_bounds, remainder_upper_bounds) s_remainder_au = (bool_remainder_selected_regions == True) bool_selected_regions = bool_selected_regions.set_selected( s_remainder_au, True) if s_ncs_related is not None and \ bool_remainder_ncs_related_mask is not None: s_ncs_related |= (bool_remainder_ncs_related_mask == True) # Now create NCS mask by eliminating all points in target (expanded) in # NCS-related copies if s_ncs_related is not None: bool_selected_regions = bool_selected_regions.set_selected( s_ncs_related, False) if tracking_data.params.map_modification.regions_to_keep is None: # Identify full (possibly expanded) ncs au starting with what we have au_mask = get_one_au(tracking_data = tracking_data, starting_mask = bool_selected_regions, removed_ncs = removed_ncs, ncs_obj = ncs_group_obj.ncs_obj, map_data = map_data, out = out) print("\nExpanding NCS AU if necessary...", file = out) print("Size of AU mask: %s Current size of AU: %s" %( au_mask.count(True), bool_selected_regions.count(True)), file = out) bool_selected_regions = (bool_selected_regions | au_mask) print("New size of AU mask: %s" %(bool_selected_regions.count(True)), file = out) sites_cart = get_marked_points_cart(mask_data = bool_selected_regions, unit_cell = ncs_group_obj.crystal_symmetry.unit_cell(), every_nth_point = tracking_data.params.segmentation.grid_spacing_for_au, boundary_radius = tracking_data.params.segmentation.radius) sites_lower_bounds, sites_upper_bounds = get_bounds_from_sites( unit_cell = ncs_group_obj.crystal_symmetry.unit_cell(), sites_cart = sites_cart, map_data = map_data) print("Original bounds: %5s %5s %5s to %5s %5s %5s" %( tuple(lower_bounds+upper_bounds)), file = out) lower_bounds = get_lower(lower_bounds, sites_lower_bounds) upper_bounds = get_upper(upper_bounds, sites_upper_bounds) print("Updated bounds: %5s %5s %5s to %5s %5s %5s" %( tuple(lower_bounds+upper_bounds)), file = out) lower_bounds, upper_bounds = adjust_bounds(params, lower_bounds, upper_bounds, map_data = map_data, out = out) box_ncs_au = params.segmentation.box_ncs_au if (not box_ncs_au): print("Using entire input map (box_ncs_au = False)", file = out) lower_bounds = map_data.origin() upper_bounds = tuple(matrix.col(map_data.all())+ matrix.col(map_data.origin())-matrix.col((1, 1, 1))) print("\nMaking two types of maps for AU of NCS mask and map with "+\ "buffer of %d grid units \nin each direction around AU" %( params.output_files.box_buffer), file = out) if params.output_files.write_output_maps: print("Both types of maps have the same origin and overlay on %s" %( os.path.join(tracking_data.params.output_files.output_directory, params.output_files.shifted_map_file)), file = out) print("\nThe standard maps (%s, %s) have the \noriginal cell dimensions." %( os.path.join(tracking_data.params.output_files.output_directory, au_mask_output_file), os.path.join(tracking_data.params.output_files.output_directory, au_map_output_file))+\ "\nThese maps show only the unique (NCS AU) part of the map.", file = out) print("\nThe cut out box_maps (%s, %s) have \nsmaller cell dimensions." %( os.path.join(tracking_data.params.output_files.output_directory, params.output_files.box_mask_file), os.path.join(tracking_data.params.output_files.output_directory, params.output_files.box_map_file), ) +\ "\nThese maps also show only the unique part of the map and have this"+\ "\nunique part cut out.\n", file = out) # Write out NCS AU with shifted origin but initial crystal_symmetry # Mask mask_data_ncs_au = get_bool_mask_as_int( ncs_group_obj = ncs_group_obj, mask_as_bool = bool_selected_regions) if au_mask_output_file and params.output_files.write_output_maps: # Write out the mask (as int) write_ccp4_map(tracking_data.crystal_symmetry, au_mask_output_file, mask_data_ncs_au) print("Output NCS AU mask: %s" %(au_mask_output_file), file = out) tracking_data.set_output_ncs_au_mask_info( file_name = au_mask_output_file, crystal_symmetry = tracking_data.crystal_symmetry, origin = mask_data_ncs_au.origin(), all = mask_data_ncs_au.all()) # Map map_data_ncs_au = map_data.deep_copy() s = (bool_selected_regions == True) mask = map_data.deep_copy() mask = mask.set_selected(s, 1) mask = mask.set_selected(~s, 0) if params.map_modification.soft_mask: # buffer and smooth the mask map_data_ncs_au, smoothed_mask_data = apply_soft_mask(map_data = map_data_ncs_au, mask_data = mask.as_double(), rad_smooth = tracking_data.params.crystal_info.resolution, crystal_symmetry = tracking_data.crystal_symmetry, out = out) half_map_data_list_au = [] for hm in half_map_data_list: # apply mask to half maps hm_data_ncs_au, hm_smoothed_mask_data = apply_soft_mask( map_data = hm.deep_copy().as_double(), mask_data = mask.as_double(), rad_smooth = tracking_data.params.crystal_info.resolution, crystal_symmetry = tracking_data.crystal_symmetry, out = out) half_map_data_list_au.append(hm_data_ncs_au) elif (box_ncs_au): # usual. If box_ncs_au is False, do not mask map_data_ncs_au = map_data_ncs_au*mask one_d = map_data_ncs_au.as_1d() n_zero = mask.count(0) n_tot = mask.size() mean_in_box = one_d.min_max_mean().mean*n_tot/(n_tot-n_zero) map_data_ncs_au = map_data_ncs_au+(1-mask)*mean_in_box half_map_data_list_au = [] for hm in half_map_data_list: # apply mask to half maps one_d = hm.as_1d() mean_in_box = one_d.min_max_mean().mean*n_tot/(n_tot-n_zero) hm_data_ncs_au = hm+(1-mask)*mean_in_box half_map_data_list_au.append(hm_data_ncs_au) del one_d, mask if au_map_output_file and params.output_files.write_output_maps: # Write out the NCS au of density write_ccp4_map(tracking_data.crystal_symmetry, au_map_output_file, map_data_ncs_au) print("Output NCS AU map: %s" %(au_map_output_file), file = out) tracking_data.set_output_ncs_au_map_info( file_name = au_map_output_file, crystal_symmetry = tracking_data.crystal_symmetry, origin = map_data_ncs_au.origin(), all = map_data_ncs_au.all()) # Now box_map of cut out AU box_mask_ncs_au, box_crystal_symmetry, \ dummy_smoothed_box_mask_data, dummy_original_box_map_data = cut_out_map( map_data = mask_data_ncs_au.as_double(), crystal_symmetry = tracking_data.crystal_symmetry, min_point = lower_bounds, max_point = upper_bounds, out = out) # Mask if params.output_files.box_mask_file and params.output_files.write_output_maps: # write out box_map NCS mask representing one AU of the NCS write_ccp4_map( box_crystal_symmetry, os.path.join(tracking_data.params.output_files.output_directory, params.output_files.box_mask_file), box_mask_ncs_au) print("Output NCS au as box (cut out) mask: %s " %( os.path.join(tracking_data.params.output_files.output_directory, params.output_files.box_mask_file)), file = out) tracking_data.set_output_box_mask_info( file_name = os.path.join(tracking_data.params.output_files.output_directory, params.output_files.box_mask_file), crystal_symmetry = box_crystal_symmetry, origin = box_mask_ncs_au.origin(), all = box_mask_ncs_au.all()) # Map box_map_ncs_au, box_crystal_symmetry, \ dummy_smoothed_box_mask_data, dummy_original_box_map_data = cut_out_map( soft_mask = tracking_data.params.map_modification.soft_mask, resolution = tracking_data.params.crystal_info.resolution, map_data = map_data_ncs_au.as_double(), crystal_symmetry = tracking_data.crystal_symmetry, min_point = lower_bounds, max_point = upper_bounds, out = out) half_map_data_list_au_box = [] for hmdlu in half_map_data_list_au: hm_box_map_ncs_au, dummy_box_crystal_symmetry, \ dummy_smoothed_box_mask_data, dummy_original_box_map_data = cut_out_map( soft_mask = tracking_data.params.map_modification.soft_mask, resolution = tracking_data.params.crystal_info.resolution, map_data = hmdlu.as_double(), crystal_symmetry = tracking_data.crystal_symmetry, min_point = lower_bounds, max_point = upper_bounds, out = out) half_map_data_list_au_box.append(hm_box_map_ncs_au) if params.control.save_box_map_ncs_au: tracking_data.set_box_map_ncs_au_map_data( box_map_ncs_au_crystal_symmetry = box_crystal_symmetry, box_map_ncs_au_map_data = box_map_ncs_au, box_mask_ncs_au_map_data = box_mask_ncs_au, box_map_ncs_au_half_map_data_list = half_map_data_list_au_box, ) if params.output_files.box_map_file: # write out NCS map as box_map (cut out region of map enclosed in box_mask) if params.output_files.write_output_maps: write_ccp4_map(box_crystal_symmetry, os.path.join(tracking_data.params.output_files.output_directory, params.output_files.box_map_file), box_map_ncs_au) print("Output NCS au as box (cut out) map: %s " %( os.path.join(tracking_data.params.output_files.output_directory, params.output_files.box_map_file)), file = out) tracking_data.set_output_box_map_info( file_name = os.path.join(tracking_data.params.output_files.output_directory, params.output_files.box_map_file), crystal_symmetry = box_crystal_symmetry, origin = box_map_ncs_au.origin(), all = box_map_ncs_au.all()) # Write out all the selected regions if params.output_files.write_output_maps: print("\nWriting out region maps. "+\ "These superimpose on the NCS AU map \nand "+\ "mask %s, %s\n" %( os.path.join(tracking_data.params.output_files.output_directory, params.output_files.box_map_file), os.path.join(tracking_data.params.output_files.output_directory, params.output_files.box_mask_file), ), file = out) map_files_written, remainder_regions_written = write_region_maps(params, map_data = map_data, tracking_data = tracking_data, ncs_group_obj = ncs_group_obj, remainder_ncs_group_obj = remainder_ncs_group_obj, out = out) # and pick up the remainder regions not already written remainder_map_files_written, dummy_remainder = write_region_maps(params, map_data = map_data, tracking_data = tracking_data, ncs_group_obj = remainder_ncs_group_obj, regions_to_skip = remainder_regions_written, out = out) map_files_written+= remainder_map_files_written else: map_files_written = [] return map_files_written def write_intermediate_maps(params, map_data = None, map_data_remaining = None, ncs_group_obj = None, tracking_data = None, out = sys.stdout): if map_data_remaining and params.output_files.remainder_map_file: write_ccp4_map( tracking_data.crystal_symmetry, params.output_files.remainder_map_file, map_data_remaining) print("Wrote output remainder map to %s" %( params.output_files.remainder_map_file), file = out) if params.segmentation.write_all_regions: for id in ncs_group_obj.selected_regions: region_mask = ncs_group_obj.edited_mask.deep_copy() s = (ncs_group_obj.edited_mask == -1) s |= (ncs_group_obj.edited_mask == id) region_mask = region_mask.set_selected(s, 1) region_mask = region_mask.set_selected(~s, 0) write_ccp4_map(tracking_data.crystal_symmetry, 'mask_%d.ccp4' %id, region_mask) print("Wrote output mask for region %d to %s" %(id, "mask_%d.ccp4" %(id)), file = out) def iterate_search(params, map_data_remaining = None, map_data = None, ncs_obj = None, ncs_group_obj = None, scattered_points = None, tracking_data = None, out = sys.stdout): # Write out intermediate maps if desired if params.output_files.write_intermediate_maps: write_intermediate_maps(params, map_data = map_data, map_data_remaining = map_data_remaining, ncs_group_obj = ncs_group_obj, tracking_data = tracking_data, out = out) new_params = deepcopy(params) new_params.segmentation.iterate_with_remainder = False new_params.segmentation.density_threshold = None new_params.output_files.write_output_maps = False new_params.output_files.output_info_file = None if params.output_files.write_intermediate_maps: new_params.output_files.au_output_file_stem = \ params.output_files.au_output_file_stem+"_cycle_2" else: new_params.output_files.au_output_file_stem = None fraction = params.segmentation.iteration_fraction if tracking_data.n_residues: new_n_residues = int(tracking_data.n_residues*fraction) new_solvent_fraction = max(0.001, min(0.999, 1- (1-tracking_data.solvent_fraction)*fraction)) new_tracking_data = deepcopy(tracking_data) if new_tracking_data.n_residues: new_tracking_data.set_n_residues(new_n_residues) new_tracking_data.set_solvent_fraction(new_solvent_fraction) new_tracking_data.set_origin_shift() # sets it to zero new_tracking_data.params.segmentation.starting_density_threshold = new_params.segmentation.starting_density_threshold # this is new print("\nIterating with remainder density", file = out) # NOTE: do not include pdb_hierarchy here unless you deep_copy it remainder_ncs_group_obj, dummy_remainder, remainder_tracking_data = run( None, params = new_params, map_data = map_data_remaining, ncs_obj = ncs_obj, target_scattered_points = scattered_points, tracking_data = new_tracking_data, is_iteration = True, out = out) if not remainder_ncs_group_obj: # Nothing to do return None # Combine the results to get remainder_id_dict # remainder_id_dict[id_remainder] = id_nearby remainder_ncs_group_obj = combine_with_iteration(params, map_data = map_data, crystal_symmetry = tracking_data.crystal_symmetry, ncs_group_obj = ncs_group_obj, remainder_ncs_group_obj = remainder_ncs_group_obj, out = out) return remainder_ncs_group_obj def bounds_overlap(lower = None, upper = None, other_lower = None, other_upper = None, tol = 1): for i in range(3): if upper[i]+toltouching_dist: continue bool_region_mask = ncs_group_obj.co.expand_mask( id_to_expand = ncs_group_obj.original_id_from_id[id], expand_size = params.segmentation.expand_size+1) # just touching s = (bool_region_mask == True) s &= (remainder_ncs_group_obj.edited_mask == id_remainder) overlaps = s.count(True) if best_overlaps is None or overlaps>best_overlaps: best_overlaps = overlaps best_id = id if best_overlaps: print("\nCombining remainder id %d with original id %d (overlaps = %d)" %( id_remainder, best_id, best_overlaps), file = out) remainder_id_dict[id_remainder] = best_id remainder_ncs_group_obj.remainder_id_dict = remainder_id_dict return remainder_ncs_group_obj def get_touching_dist(centers, default = 100., min_dist = 8.): mean_dist = 0. mean_dist_n = 0. nskip = max(1, len(centers)//10) # try to get 10 for i in range(0, len(centers), nskip): if i == 0: target = centers[1:] elif i == len(centers)-1: target = centers[:-1] else: target = centers[:i] target.extend(centers[i+1:]) other = centers[i:i+1] if not target or not other: continue dist = get_closest_dist(target, other) if dist is not None: mean_dist+= dist mean_dist_n+= 1. if mean_dist_n>0: return max(min_dist, 2.0*mean_dist/mean_dist_n) else: return default def get_grid_units(map_data = None, crystal_symmetry = None, radius = None, out = sys.stdout): N_ = map_data.all() sx, sy, sz = 1/N_[0], 1/N_[1], 1/N_[2] sx_cart, sy_cart, sz_cart = crystal_symmetry.unit_cell().orthogonalize( [sx, sy, sz]) grid_spacing = (sx_cart+sy_cart+sz_cart)/3. grid_units = int(radius/grid_spacing) min_cell_grid_units = min(N_[0], N_[1], N_[2]) grid_units = max(1, min(grid_units, int(min_cell_grid_units/3))) print("Grid units representing %7.1f A will be %d" %( radius, grid_units), file = out) return grid_units def cut_out_map(map_data = None, crystal_symmetry = None, soft_mask = None, soft_mask_radius = None, resolution = None, shift_origin = None, min_point = None, max_point = None, out = sys.stdout): from cctbx import uctbx from cctbx import maptbx na = map_data.all() # tuple with dimensions for i in range(3): assert min_point[i] >= 0 assert max_point[i] < na[i] # 2019-11-05 just na-1 new_map_data = maptbx.copy(map_data, tuple(min_point), tuple(max_point)) # NOTE: end point of map is max_point, so size of map (new all()) is # (max_point-min_point+ (1, 1, 1)) # shrink unit cell, angles are the same # NOTE 2: the origin of output map will be min_point (not 0, 0, 0). shrunk_uc = [] for i in range(3): shrunk_uc.append( crystal_symmetry.unit_cell().parameters()[i]*new_map_data.all()[i]/na[i] ) uc_params = crystal_symmetry.unit_cell().parameters() new_unit_cell_box = uctbx.unit_cell( parameters = (shrunk_uc[0], shrunk_uc[1], shrunk_uc[2], uc_params[3], uc_params[4], uc_params[5])) new_crystal_symmetry = crystal.symmetry( unit_cell = new_unit_cell_box, space_group = 'p1') if soft_mask: if soft_mask_radius is None: soft_mask_radius = resolution assert soft_mask_radius is not None original_map_data = new_map_data.deep_copy() new_map_data, smoothed_mask_data = set_up_and_apply_soft_mask( map_data = new_map_data, shift_origin = shift_origin, crystal_symmetry = new_crystal_symmetry, resolution = resolution, radius = soft_mask_radius, out = out) else: original_map_data = None smoothed_mask_data = None return new_map_data, new_crystal_symmetry, \ smoothed_mask_data, original_map_data def get_zero_boundary_map( map_data = None, grid_units_for_boundary = None, crystal_symmetry = None, radius = None): assert grid_units_for_boundary or (crystal_symmetry and radius) # grid_units is how many grid units are about equal to soft_mask_radius if grid_units_for_boundary is None: grid_units = get_grid_units(map_data = map_data, crystal_symmetry = crystal_symmetry, radius = radius, out = null_out()) grid_units = int(0.5+0.5*grid_units) else: grid_units = grid_units_for_boundary from cctbx import maptbx zero_boundary_map = maptbx.zero_boundary_box_map( map_data, grid_units).result() return zero_boundary_map def set_up_and_apply_soft_mask(map_data = None, shift_origin = None, crystal_symmetry = None, resolution = None, grid_units_for_boundary = None, radius = None, out = None): if out is None: from libtbx.utils import null_out out = null_out() acc = map_data.accessor() map_data = map_data.shift_origin() new_acc = map_data.accessor() # Add soft boundary to mean around outside of mask zero_boundary_map = get_zero_boundary_map( map_data = map_data, grid_units_for_boundary = grid_units_for_boundary, crystal_symmetry = crystal_symmetry, radius = radius) # this map is zero's around the edge and 1 in the middle # multiply zero_boundary_map--smoothed & new_map_data and return print("Applying soft mask to boundary of cut out map", file = out) new_map_data, smoothed_mask_data = apply_soft_mask(map_data = map_data, mask_data = zero_boundary_map, rad_smooth = resolution, crystal_symmetry = crystal_symmetry, out = out) if new_acc != acc: new_map_data.reshape(acc) smoothed_mask_data.reshape(acc) return new_map_data, smoothed_mask_data def apply_shift_to_pdb_hierarchy( origin_shift = None, crystal_symmetry = None, pdb_hierarchy = None, out = sys.stdout): if origin_shift is not None: sites_cart = pdb_hierarchy.atoms().extract_xyz() sites_cart_shifted = sites_cart+\ flex.vec3_double(sites_cart.size(), origin_shift) pdb_hierarchy.atoms().set_xyz(sites_cart_shifted) return pdb_hierarchy def apply_origin_shift(origin_shift = None, ncs_object = None, shifted_ncs_object = None, pdb_hierarchy = None, target_hierarchy = None, map_data = None, shifted_map_file = None, shifted_pdb_file = None, shifted_ncs_file = None, tracking_data = None, sharpening_target_pdb_inp = None, out = sys.stdout): if shifted_map_file: write_ccp4_map(tracking_data.crystal_symmetry, shifted_map_file, map_data) print("Wrote shifted map to %s" %( shifted_map_file), file = out) tracking_data.set_shifted_map_info(file_name = shifted_map_file, crystal_symmetry = tracking_data.crystal_symmetry, origin = map_data.origin(), all = map_data.all()) if origin_shift: # Note origin shift does not change crystal_symmetry if pdb_hierarchy: pdb_hierarchy = apply_shift_to_pdb_hierarchy( origin_shift = origin_shift, crystal_symmetry = tracking_data.crystal_symmetry, pdb_hierarchy = pdb_hierarchy, out = out) if sharpening_target_pdb_inp: sharpening_target_pdb_inp = apply_shift_to_pdb_hierarchy( origin_shift = origin_shift, crystal_symmetry = tracking_data.crystal_symmetry, pdb_hierarchy = sharpening_target_pdb_inp.construct_hierarchy(), out = out).as_pdb_input() if target_hierarchy: target_hierarchy = apply_shift_to_pdb_hierarchy( origin_shift = origin_shift, crystal_symmetry = tracking_data.crystal_symmetry, pdb_hierarchy = target_hierarchy, out = out) from scitbx.math import matrix if ncs_object and not shifted_ncs_object: shfted_ncs_object = ncs_object.coordinate_offset( coordinate_offset = matrix.col(origin_shift)) if shifted_pdb_file and pdb_hierarchy: import iotbx.pdb f = open(shifted_pdb_file, 'w') print(iotbx.pdb.format_cryst1_record( crystal_symmetry = tracking_data.crystal_symmetry), file = f) print(pdb_hierarchy.as_pdb_string(), file = f) f.close() print("Wrote shifted pdb file to %s" %( shifted_pdb_file), file = out) tracking_data.set_shifted_pdb_info(file_name = shifted_pdb_file, n_residues = pdb_hierarchy.overall_counts().n_residues) if shifted_ncs_file and shifted_ncs_object: shifted_ncs_object.format_all_for_group_specification( file_name = shifted_ncs_file) print("Wrote %s NCS operators for shifted map to %s" %( shifted_ncs_object.max_operators(), shifted_ncs_file), file = out) if tracking_data.input_ncs_info.has_updated_operators(): print("NOTE: these may include additional operators added to fill the cell"+\ " or\nhave fewer operators if not all applied.", file = out) tracking_data.set_shifted_ncs_info(file_name = shifted_ncs_file, number_of_operators = shifted_ncs_object.max_operators(), is_helical_symmetry = tracking_data.input_ncs_info.is_helical_symmetry) tracking_data.shifted_ncs_info.show_summary(out = out) return shifted_ncs_object, pdb_hierarchy, target_hierarchy, tracking_data, \ sharpening_target_pdb_inp def restore_pdb(params, tracking_data = None, out = sys.stdout): if not params.output_files.restored_pdb: params.output_files.restored_pdb = \ params.input_files.pdb_to_restore[:-4]+"_restored.pdb" print("Shifting origin of %s and writing to %s" %( params.input_files.pdb_to_restore, params.output_files.restored_pdb), file = out) os = tracking_data.origin_shift origin_shift = (-os[0], -os[1], -os[2]) print("Origin shift will be: %.1f %.1f %.1f "%(origin_shift), file = out) import iotbx.pdb pdb_inp = iotbx.pdb.input(file_name = params.input_files.pdb_to_restore) pdb_hierarchy = pdb_inp.construct_hierarchy() pdb_hierarchy = apply_shift_to_pdb_hierarchy( origin_shift = origin_shift, crystal_symmetry = tracking_data.crystal_symmetry, pdb_hierarchy = pdb_hierarchy, out = out) f = open(params.output_files.restored_pdb, 'w') print(iotbx.pdb.format_cryst1_record( crystal_symmetry = tracking_data.crystal_symmetry), file = f) print(pdb_hierarchy.as_pdb_string(), file = f) f.close() print("Wrote restored pdb file to %s" %( params.output_files.restored_pdb), file = out) def find_threshold_in_map(target_points = None, map_data = None, require_at_least_target_points = None, iter_max = 10): map_1d = map_data.as_1d() map_mean = map_1d.min_max_mean().mean map_max = map_1d.min_max_mean().max map_min = map_1d.min_max_mean().min cutoff = map_mean low = map_min high = map_max best_cutoff = None best_score = None for iter in range(iter_max): s = (map_1d >cutoff) n_cutoff = s.count(True) if (not require_at_least_target_points) or (n_cutoff >= target_points): score = abs(n_cutoff-target_points) else: score = 1.e+10 # allow it but anything above cutoff will be better if best_score is None or score < best_score: best_cutoff = cutoff best_score = score if n_cutoff == target_points: return best_cutoff elif n_cutoff < target_points: # lower it high = cutoff cutoff = 0.5*(cutoff+low) else: # raise it low = cutoff cutoff = 0.5*(cutoff+high) return best_cutoff def remove_points(mask, remove_points = None): keep_points = (remove_points == False) new_mask = (mask & keep_points) return new_mask def get_ncs_sites_cart(sites_cart = None, ncs_id = None, ncs_obj = None, unit_cell = None, ncs_in_cell_only = True): ncs_sites_cart = flex.vec3_double() if not ncs_obj or not ncs_obj.ncs_groups() or not ncs_obj.ncs_groups()[0] or \ not ncs_obj.ncs_groups()[0].translations_orth(): return ncs_sites_cart # identify ncs-related points ncs_group = ncs_obj.ncs_groups()[0] identity_op = ncs_group.identity_op_id() ncs_sites_cart = flex.vec3_double() for xyz_cart in sites_cart: for i0 in range(len(ncs_group.translations_orth())): if i0 == identity_op: continue if ncs_id is not None and i0!= ncs_id: continue r = ncs_group.rota_matrices_inv()[i0] # inverse maps pos 0 on to pos i t = ncs_group.translations_orth_inv()[i0] new_xyz_cart = r * matrix.col(xyz_cart) + t ncs_sites_cart.append(new_xyz_cart) if ncs_in_cell_only: new_sites_cart = flex.vec3_double() ncs_sites_frac = unit_cell.fractionalize(ncs_sites_cart) for site_frac, site_cart in zip(ncs_sites_frac, ncs_sites_cart): if site_frac[0]>= 0 and site_frac[0]<= 1 and \ site_frac[1]>= 0 and site_frac[1]<= 1 and \ site_frac[2]>= 0 and site_frac[2]<= 1: new_sites_cart.append(site_cart) ncs_sites_cart = new_sites_cart return ncs_sites_cart def get_ncs_mask(map_data = None, unit_cell = None, ncs_object = None, starting_mask = None, radius = None, expand_radius = None, overall_mask = None, every_nth_point = None): assert every_nth_point is not None if not expand_radius: expand_radius = 2.*radius working_au_mask = starting_mask.deep_copy() working_ncs_mask = mask_from_sites_and_map( # empty ncs mask map_data = map_data, unit_cell = unit_cell, sites_cart = flex.vec3_double(), radius = radius, overall_mask = overall_mask) au_points_last = working_au_mask.count(True) ncs_points_last = working_ncs_mask.count(True) max_tries = 10000 for ii in range(max_tries): # just a big number; should take just a few # Find all points in au (sample every_nth_point in grid) au_sites_cart = get_marked_points_cart(mask_data = working_au_mask, unit_cell = unit_cell, every_nth_point = every_nth_point, boundary_radius = radius) # Find all points ncs-related to marked point in mask ncs_sites_cart = get_ncs_sites_cart(sites_cart = au_sites_cart, ncs_obj = ncs_object, unit_cell = unit_cell, ncs_in_cell_only = True) # Expand au slightly with all points near to au_sites_cart new_au_mask = mask_from_sites_and_map( map_data = map_data, unit_cell = unit_cell, sites_cart = au_sites_cart, radius = radius, overall_mask = overall_mask) working_au_mask = (working_au_mask | new_au_mask) # add on to existing keep_points = (working_ncs_mask == False) # cross off those in ncs working_au_mask = (working_au_mask & keep_points) # mark ncs au with all points not in au that are close to ncs_sites_cart new_ncs_mask = mask_from_sites_and_map( map_data = map_data, unit_cell = unit_cell, sites_cart = ncs_sites_cart, radius = radius, overall_mask = overall_mask) keep_points = (working_au_mask == False) # cross off those in au new_ncs_mask = (new_ncs_mask & keep_points) working_ncs_mask = (new_ncs_mask | working_ncs_mask) # add on to existing au_points = working_au_mask.count(True) ncs_points = working_ncs_mask.count(True) if au_points == au_points_last and ncs_points == ncs_points_last: break au_points_last = au_points ncs_points_last = ncs_points # Now expand the au and repeat working_au_mask = mask_from_sites_and_map( map_data = map_data, unit_cell = unit_cell, sites_cart = au_sites_cart, radius = expand_radius, overall_mask = overall_mask) keep_points = (working_ncs_mask == False) # cross off those in ncs working_au_mask = (working_au_mask & keep_points) return working_au_mask, working_ncs_mask def renormalize_map_data( map_data = None, solvent_fraction = None): sd = max(0.0001, map_data.sample_standard_deviation()) if solvent_fraction >= 10.: solvent_fraction = solvent_fraction/100. solvent_fraction = min(0.999, max(0.001, solvent_fraction)) scaled_sd = sd/(1-solvent_fraction)**0.5 map_data = (map_data-map_data.as_1d().min_max_mean().mean)/scaled_sd return map_data def mask_from_sites_and_map( map_data = None, unit_cell = None, sites_cart = None, radius = None, overall_mask = None): assert radius is not None from cctbx import maptbx sel = maptbx.grid_indices_around_sites( unit_cell = unit_cell, fft_n_real = map_data.focus(), fft_m_real = map_data.all(), sites_cart = sites_cart, site_radii = flex.double(sites_cart.size(), radius)) map_data_1d = map_data.as_1d() mask = (map_data_1d == 0 and map_data_1d == 1) # 1D bool array all False mask.set_selected(sel, True) # mark points around sites mask.reshape(map_data.accessor()) if overall_mask: assert overall_mask.all() == mask.all() mask = (mask & overall_mask) return mask def set_radius(unit_cell = None, map_data = None, every_nth_point = None): # Set radius so that radius will capture all points on grid if sampled # on every_nth_point a, b, c = unit_cell.parameters()[:3] nx, ny, nz = map_data.all() # furthest possible minimum distance between grid points max_diagonal_between_sampled = every_nth_point*( (a/nx)**2+(b/ny)**2+(c/nz)**2)**0.5 radius = max_diagonal_between_sampled*0.55 # big enough to cover everything return radius def get_marked_points_cart(mask_data = None, unit_cell = None, every_nth_point = 3, boundary_radius = None): # return list of cartesian coordinates of grid points that are marked # only sample every every_nth_point in each direction... assert mask_data.origin() == (0, 0, 0) nx, ny, nz = mask_data.all() if boundary_radius: # How far from edges shall we stay: grid_frac = (1./nx, 1./ny, 1./nz) grid_orth = unit_cell.orthogonalize(grid_frac) boundary_grid_points = 0 for go in grid_orth: bgp = int(0.99+boundary_radius/go) boundary_grid_points = max(boundary_grid_points, bgp) else: boundary_grid_points = 0 marked_points = maptbx.marked_grid_points( map_data = mask_data, every_nth_point = every_nth_point).result() sites_frac = flex.vec3_double() boundary_points_skipped = 0 for grid_point in marked_points: if boundary_grid_points: if \ grid_point[0]nx-boundary_grid_points or \ grid_point[1]ny-boundary_grid_points or \ grid_point[2]nz-boundary_grid_points: # XXX was typo previously boundary_points_skipped+= 1 continue sites_frac.append( (grid_point[0]/nx, grid_point[1]/ny, grid_point[2]/nz)) sites_cart = unit_cell.orthogonalize(sites_frac) return sites_cart def get_overall_mask( map_data = None, mask_threshold = None, fraction_of_max_mask_threshold = None, use_solvent_content_for_threshold = None, # use instead of fraction_of mask_padding_fraction = None, solvent_fraction = None, crystal_symmetry = None, radius = None, resolution = None, d_max = 100000., out = sys.stdout): # This routine cannot use mask_data with origin != (0,0,0) if map_data.origin() != (0,0,0): print("Map origin must be at (0,0,0) for get_overall_mask") assert map_data.origin() == (0,0,0) # Map origin must be at (0,0,0) # Make a local SD map from our map-data from cctbx.maptbx import crystal_gridding from cctbx import sgtbx cg = crystal_gridding( unit_cell = crystal_symmetry.unit_cell(), space_group_info = sgtbx.space_group_info(number = 1), # Always pre_determined_n_real = map_data.all()) if not resolution: from cctbx.maptbx import d_min_from_map resolution = d_min_from_map( map_data, crystal_symmetry.unit_cell(), resolution_factor = 1./4.) print("\nEstimated resolution of map: %6.1f A\n" %( resolution), file = out) if radius: smoothing_radius = 2.*radius else: smoothing_radius = 2.*resolution from iotbx.map_manager import map_manager mm = map_manager(map_data = map_data, unit_cell_grid = map_data.all(), unit_cell_crystal_symmetry = crystal_symmetry, wrapping = False) map_coeffs = mm.map_as_fourier_coefficients( d_min = resolution, d_max = d_max) if not map_coeffs: raise Sorry("No map coeffs obtained") complete_set = map_coeffs.complete_set() stol = flex.sqrt(complete_set.sin_theta_over_lambda_sq().data()) import math w = 4 * stol * math.pi * smoothing_radius sphere_reciprocal = 3 * (flex.sin(w) - w * flex.cos(w))/flex.pow(w, 3) try: temp = complete_set.structure_factors_from_map( flex.pow2(map_data-map_data.as_1d().min_max_mean().mean)) except Exception as e: print(e, file = out) print ("The sampling of the map appears to be too low for a "+ "\nresolution of %s. Using a larger value for resolution" %( resolution), file = out) from cctbx.maptbx import d_min_from_map resolution = d_min_from_map( map_data, crystal_symmetry.unit_cell(), resolution_factor = 1./4.) print("\nEstimated resolution of map: %6.1f A\n" %( resolution), file = out) map_coeffs = map_coeffs.resolution_filter(d_min = resolution, d_max = d_max) complete_set = map_coeffs.complete_set() stol = flex.sqrt(complete_set.sin_theta_over_lambda_sq().data()) import math w = 4 * stol * math.pi * smoothing_radius sphere_reciprocal = 3 * (flex.sin(w) - w * flex.cos(w))/flex.pow(w, 3) temp = complete_set.structure_factors_from_map( flex.pow2(map_data-map_data.as_1d().min_max_mean().mean)) fourier_coeff = complete_set.array(data = temp.data()*sphere_reciprocal) sd_map = fourier_coeff.fft_map( crystal_gridding = cg, ).apply_volume_scaling().real_map_unpadded() assert sd_map.all() == map_data.all() # now use sd_map # First mask out the map based on threshold mm = sd_map.as_1d().min_max_mean() max_in_sd_map = mm.max mean_in_map = mm.mean min_in_map = mm.min print("Highest value in SD map is %7.2f. Mean is %7.2f . Lowest is %7.2f " %( max_in_sd_map, mean_in_map, min_in_map), file = out) if fraction_of_max_mask_threshold and ( (not solvent_fraction) or (not use_solvent_content_for_threshold)): mask_threshold = fraction_of_max_mask_threshold*max_in_sd_map print("Using fraction of max as threshold: %.3f " %( fraction_of_max_mask_threshold), \ "which is threshold of %.3f" %(mask_threshold), file = out) if mask_padding_fraction: # Adjust threshold to increase by mask_padding_fraction, proportional # to fraction available overall_mask = (sd_map>= mask_threshold) current_above_threshold = overall_mask.count(True)/overall_mask.size() # current+(1-current)*pad additional_padding = (1-current_above_threshold)*mask_padding_fraction target_above_threshold = min( 0.99, current_above_threshold+additional_padding) print("Target with padding of %.2f will be %.2f" %( mask_padding_fraction, target_above_threshold), file = out) solvent_fraction = (1-target_above_threshold) mask_threshold = None if mask_threshold: print("Cutoff for mask will be input threshold", file = out) threshold = mask_threshold else: # guess based on solvent_fraction if solvent_fraction is None: print("Guessing solvent fraction of 0.9", file = out) solvent_fraction = 0.9 # just guess threshold = find_threshold_in_map(target_points = int( (1.-solvent_fraction)*sd_map.size()), map_data = sd_map) print("Cutoff will be threshold marking about %7.1f%% of cell" %( 100.*(1.-solvent_fraction)), file = out) overall_mask = (sd_map>= threshold) print("Model region of map "+\ "(density above %7.3f )" %( threshold) +" includes %7.1f%% of map" %( 100.*overall_mask.count(True)/overall_mask.size()), file = out) return overall_mask, max_in_sd_map, sd_map def get_skew(data = None): mean = data.min_max_mean().mean sd = data.standard_deviation_of_the_sample() x = data-mean return (x**3).min_max_mean().mean/sd**3 def get_kurtosis(data = None): mean = data.min_max_mean().mean sd = data.standard_deviation_of_the_sample() x = data-mean return (x**4).min_max_mean().mean/sd**4 def score_map(map_data = None, sharpening_info_obj = None, solvent_fraction = None, fraction_occupied = None, wrapping = None, sa_percent = None, region_weight = None, max_regions_to_test = None, scale_region_weight = False, out = sys.stdout): if sharpening_info_obj: solvent_fraction = sharpening_info_obj.solvent_fraction wrapping = sharpening_info_obj.wrapping fraction_occupied = sharpening_info_obj.fraction_occupied sa_percent = sharpening_info_obj.sa_percent region_weight = sharpening_info_obj.region_weight max_regions_to_test = sharpening_info_obj.max_regions_to_test else: sharpening_info_obj = sharpening_info() if solvent_fraction is None: # skip SA score sharpening_info_obj.adjusted_sa = 0. assert sharpening_info_obj.sharpening_target == 'kurtosis' else: # usual map_data = renormalize_map_data( map_data = map_data, solvent_fraction = solvent_fraction) target_in_all_regions = map_data.size()*fraction_occupied*(1-solvent_fraction) print("\nTarget number of points in all regions: %.0f" %( target_in_all_regions), file = out) threshold = find_threshold_in_map(target_points = int( target_in_all_regions), map_data = map_data) print("Cutoff will be threshold of %7.2f marking %7.1f%% of cell" %( threshold, 100.*(1.-solvent_fraction)), file = out) co, sorted_by_volume, min_b, max_b = get_co( map_data = map_data.deep_copy(), threshold = threshold, wrapping = wrapping) if len(sorted_by_volume)<2: return sharpening_info_obj# skip it, nothing to do target_sum = sa_percent* target_in_all_regions*0.01 print("Points for %.1f percent of target in all regions: %.1f" %( sa_percent, target_sum), file = out) cntr = 0 sum_v = 0. sum_new_v = 0. for p in sorted_by_volume[1:max_regions_to_test+2]: cntr+= 1 v, i = p sum_v+= v bool_expanded = co.expand_mask(id_to_expand = i, expand_size = 1) new_v = bool_expanded.count(True)-v sum_new_v+= new_v sa_ratio = new_v/v if sum_v>= target_sum: break sa_ratio = sum_new_v/max(1., sum_v) # ratio of SA to volume regions = len(sorted_by_volume[1:]) normalized_regions = regions/max(1, target_in_all_regions) skew = get_skew(map_data.as_1d()) if scale_region_weight: solvent_fraction_std = 0.85 # typical value, ends up as scale on weight region_weight_scale = (1.-solvent_fraction)/(1.-solvent_fraction_std) region_weight_use = region_weight*region_weight_scale else: region_weight_use = region_weight sharpening_info_obj.adjusted_sa = \ sa_ratio - region_weight_use*normalized_regions sharpening_info_obj.sa_ratio = sa_ratio sharpening_info_obj.normalized_regions = normalized_regions sharpening_info_obj.kurtosis = get_kurtosis(map_data.as_1d()) if sharpening_info_obj.sharpening_target == 'adjusted_path_length': sharpening_info_obj.adjusted_path_length = get_adjusted_path_length( map_data = map_data, resolution = sharpening_info_obj.resolution, crystal_symmetry = sharpening_info_obj.crystal_symmetry, out = out) else: sharpening_info_obj.adjusted_path_length = None if sharpening_info_obj.sharpening_target == 'kurtosis': sharpening_info_obj.score = sharpening_info_obj.kurtosis if sharpening_info_obj.sharpening_target == 'adjusted_path_length': sharpening_info_obj.score = sharpening_info_obj.adjusted_path_length else: sharpening_info_obj.score = sharpening_info_obj.adjusted_sa return sharpening_info_obj def get_adjusted_path_length( map_data = None, crystal_symmetry = None, resolution = None, out = sys.stdout): try: from phenix.autosol.trace_and_build import trace_and_build except Exception as e: # Not available return 0 from phenix.programs.trace_and_build import master_phil_str import iotbx.phil tnb_params = iotbx.phil.parse(master_phil_str).extract() tnb_params.crystal_info.resolution = resolution tnb_params.strategy.retry_long_branches = False tnb_params.strategy.correct_segments = False tnb_params.strategy.split_and_join = False tnb_params.strategy.vary_sharpening = [] tnb_params.strategy.get_path_length_only = True tnb_params.trace_and_build.find_helices_strands = False tnb = trace_and_build( params = tnb_params, map_data = map_data, crystal_symmetry = crystal_symmetry, origin_cart = (0, 0, 0), origin = (0, 0, 0), log = out) tnb.run() return tnb.adjusted_path_length def sharpen_map_with_si(sharpening_info_obj = None, f_array_normalized = None, f_array = None, phases = None, map_data = None, overall_b = None, resolution = None, out = sys.stdout): si = sharpening_info_obj if si.sharpening_method == 'no_sharpening': return map_and_b_object(map_data = map_data) if map_data and (not f_array or not phases): map_coeffs, dummy = get_f_phases_from_map(map_data = map_data, crystal_symmetry = si.crystal_symmetry, d_min = si.resolution, d_min_ratio = si.d_min_ratio, return_as_map_coeffs = True, scale_max = si.scale_max, out = out) f_array, phases = map_coeffs_as_fp_phi(map_coeffs) if si.remove_aniso: if si.use_local_aniso and \ (si.local_aniso_in_local_sharpening or (si.local_aniso_in_local_sharpening is None and si.ncs_copies == 1)) and \ si.original_aniso_obj: # use original aniso_obj = si.original_aniso_obj print("\nRemoving aniso from map using saved aniso object before sharpening\n", file = out) else: print("\nRemoving aniso from map before sharpening\n", file = out) aniso_obj = None from cctbx.maptbx.refine_sharpening import analyze_aniso f_array, f_array_ra = analyze_aniso( aniso_obj = aniso_obj, remove_aniso = si.remove_aniso, f_array = f_array, resolution = si.resolution, out = out) if si.is_model_sharpening() or si.is_half_map_sharpening(): from cctbx.maptbx.refine_sharpening import scale_amplitudes ff = f_array.phase_transfer(phase_source = phases, deg = True) map_and_b = scale_amplitudes( map_coeffs = f_array.phase_transfer(phase_source = phases, deg = True), si = si, overall_b = overall_b, out = out) return map_and_b elif si.is_resolution_dependent_sharpening(): if f_array_normalized is None: from cctbx.maptbx.refine_sharpening import get_sharpened_map, \ quasi_normalize_structure_factors (d_max, d_min) = f_array.d_max_min() if not f_array.binner(): f_array.setup_binner(n_bins = si.n_bins, d_max = d_max, d_min = d_min) f_array_normalized = quasi_normalize_structure_factors( f_array, set_to_minimum = 0.01) map_data = get_sharpened_map(ma = f_array_normalized, phases = phases, b = si.resolution_dependent_b, resolution = si.resolution, n_real = si.n_real, d_min_ratio = si.d_min_ratio) return map_and_b_object(map_data = map_data) else: map_and_b = apply_sharpening(n_real = si.n_real, f_array = f_array, phases = phases, sharpening_info_obj = si, crystal_symmetry = si.crystal_symmetry, out = null_out()) return map_and_b def put_bounds_in_range( lower_bounds = None, upper_bounds = None, box_size = None, n_buffer = None, n_real = None, out = sys.stdout): # put lower and upper inside (0, n_real-1) and try to make size at least minimum new_lb = [] new_ub = [] print("Putting bounds in range...(%s, %s, %s) to (%s, %s, %s)" %( tuple(list(lower_bounds)+list(upper_bounds))), file = out) if n_buffer: print("Buffer of %s added" %(n_buffer), file = out) for lb, ub, ms, nr in zip(lower_bounds, upper_bounds, box_size, n_real): if ubub: lb = ub extra = (ms-(ub-lb))//2 lb = lb-extra ub = ub+extra if n_buffer: lb = lb-n_buffer ub = ub+n_buffer if lb<0: shift = -lb lb+= shift ub+= shift boundary = int(ms-(ub-lb+1))//2 if boundary>0: lb = lb-boundary ub = ub+boundary if lb<0: lb = 0 if ub>= nr: ub = nr-1 # 2019-11-05 cannot go beyond na-1 new_lb.append(lb) new_ub.append(ub) print("New bounds ...(%s, %s, %s) to (%s, %s, %s)" %( tuple(list(new_lb)+list(new_ub))), file = out) return tuple(new_lb), tuple(new_ub) def get_iterated_solvent_fraction(map = None, verbose = None, resolve_size = None, crystal_symmetry = None, mask_padding_fraction = None, fraction_of_max_mask_threshold = None, solvent_content = None, use_solvent_content_for_threshold = None, # use instead of fraction_of_.. cell_cutoff_for_solvent_from_mask = None, mask_resolution = None, return_mask_and_solvent_fraction = None, out = sys.stdout): if cell_cutoff_for_solvent_from_mask and \ crystal_symmetry.unit_cell().volume() > cell_cutoff_for_solvent_from_mask**3: #go directly to low_res_mask return get_solvent_fraction_from_low_res_mask( crystal_symmetry = crystal_symmetry, map_data = map.deep_copy(), mask_padding_fraction = mask_padding_fraction, fraction_of_max_mask_threshold = fraction_of_max_mask_threshold, solvent_content = solvent_content, use_solvent_content_for_threshold = use_solvent_content_for_threshold, return_mask_and_solvent_fraction = return_mask_and_solvent_fraction, mask_resolution = mask_resolution, out = out) try: from phenix.autosol.map_to_model import iterated_solvent_fraction solvent_fraction, overall_mask = iterated_solvent_fraction( crystal_symmetry = crystal_symmetry, map_as_double = map, verbose = verbose, resolve_size = resolve_size, out = out) if solvent_fraction<= 0.989: # means that it was 0.99 which is hard limit if return_mask_and_solvent_fraction: return overall_mask, solvent_fraction else: return solvent_fraction else: # use backup method return get_solvent_fraction_from_low_res_mask( crystal_symmetry = crystal_symmetry, map_data = map.deep_copy(), mask_padding_fraction = mask_padding_fraction, fraction_of_max_mask_threshold = fraction_of_max_mask_threshold, solvent_content = solvent_content, use_solvent_content_for_threshold = use_solvent_content_for_threshold, return_mask_and_solvent_fraction = return_mask_and_solvent_fraction, mask_resolution = mask_resolution, out = out) except Exception as e: # catch case where map was not on proper grid if str(e).find("sym equiv of a grid point must be a grid point")>-1: print("\nSpace group:%s \n Unit cell: %s \n Gridding: %s \nError message: %s" %( crystal_symmetry.space_group().info(), str(crystal_symmetry.unit_cell().parameters()), str(map.all()), str(e)), file = out) raise Sorry( "The input map seems to be on a grid incompatible with crystal symmetry"+ "\n(symmetry equivalents of a grid point must be on "+ "an integer grid point)") elif str(e).find("maximum size for resolve is")>-1: raise Sorry(str(e)+ "\nIt may be possible to go on by supplying solvent content"+ "or molecular_mass") # Try to get solvent fraction with low_res mask return get_solvent_fraction_from_low_res_mask( crystal_symmetry = crystal_symmetry, map_data = map.deep_copy(), mask_padding_fraction = mask_padding_fraction, fraction_of_max_mask_threshold = fraction_of_max_mask_threshold, solvent_content = solvent_content, use_solvent_content_for_threshold = use_solvent_content_for_threshold, return_mask_and_solvent_fraction = return_mask_and_solvent_fraction, mask_resolution = mask_resolution, out = out) def get_solvent_fraction_from_low_res_mask( crystal_symmetry = None, map_data = None, fraction_of_max_mask_threshold = None, solvent_content = None, use_solvent_content_for_threshold = None, # use instead of fraction_of mask_padding_fraction = None, return_mask_and_solvent_fraction = None, mask_resolution = None, out = sys.stdout): overall_mask, max_in_sd_map, sd_map = get_overall_mask(map_data = map_data, fraction_of_max_mask_threshold = fraction_of_max_mask_threshold, use_solvent_content_for_threshold = use_solvent_content_for_threshold, mask_padding_fraction = mask_padding_fraction, solvent_fraction = solvent_content, # note name change XXX crystal_symmetry = crystal_symmetry, resolution = mask_resolution, out = out) if overall_mask is None: if return_mask_and_solvent_fraction: return None, None else: return None solvent_fraction = overall_mask.count(False)/overall_mask.size() print("Solvent fraction from overall mask: %.3f " %(solvent_fraction), file = out) if return_mask_and_solvent_fraction: mask_data = map_data.deep_copy() mask_data.as_1d().set_selected(overall_mask.as_1d(), 1) mask_data.as_1d().set_selected(~overall_mask.as_1d(), 0) return mask_data, solvent_fraction else: return solvent_fraction def get_solvent_fraction_from_molecular_mass( do_not_adjust_dalton_scale = None, crystal_symmetry = None, molecular_mass = None, out = sys.stdout): map_volume = crystal_symmetry.unit_cell().volume() density_factor = 1000*1.23 # just protein density, close enough... mm = molecular_mass molecule_fraction = mm*density_factor/map_volume if do_not_adjust_dalton_scale or molecule_fraction > 1 and mm > 1000: mm = mm/1000 # was in Da solvent_fraction = max(0.01, min(1., 1 - ( mm*density_factor/map_volume))) print("Solvent content of %7.2f from molecular mass of %7.1f kDa" %( solvent_fraction, mm), file = out) return solvent_fraction def set_up_si(var_dict = None, crystal_symmetry = None, is_crystal = None, ncs_copies = None, n_residues = None, solvent_fraction = None, molecular_mass = None, pdb_inp = None, map = None, auto_sharpen = True, half_map_data_list = None, verbose = None, out = sys.stdout): si = sharpening_info(n_real = map.all()) args = [] auto_sharpen_methods = var_dict.get('auto_sharpen_methods') if auto_sharpen_methods and auto_sharpen_methods != ['None'] and \ len(auto_sharpen_methods) == 1: sharpening_method = auto_sharpen_methods[0] else: sharpening_method = None for param in [ 'verbose', 'resolve_size', 'seq_file', 'sequence', 'box_size', 'target_n_overlap', 'restrict_map_size', 'box_center', 'remove_aniso', 'input_weight_map_pickle_file', 'output_weight_map_pickle_file', 'read_sharpened_maps', 'write_sharpened_maps', 'select_sharpened_map', 'output_directory', 'smoothing_radius', 'use_local_aniso', 'local_aniso_in_local_sharpening', 'overall_before_local', 'local_sharpening', 'box_in_auto_sharpen', 'density_select_in_auto_sharpen', 'density_select_threshold_in_auto_sharpen', 'use_weak_density', 'resolution', 'd_min_ratio', 'scale_max', 'input_d_cut', 'b_blur_hires', 'discard_if_worse', 'mask_atoms', 'mask_atoms_atom_radius', 'value_outside_atoms', 'soft_mask', 'tol_r', 'abs_tol_t', 'rel_tol_t', 'require_helical_or_point_group_symmetry', 'max_helical_operators', 'allow_box_if_b_iso_set', 'max_box_fraction', 'cc_cut', 'max_cc_for_rescale', 'scale_using_last', 'density_select_max_box_fraction', 'k_sharpen', 'optimize_b_blur_hires', 'iterate', 'optimize_d_cut', 'residual_target', 'sharpening_target', 'search_b_min', 'search_b_max', 'search_b_n', 'adjust_region_weight', 'region_weight_method', 'region_weight_factor', 'region_weight_buffer', 'region_weight_default', 'target_b_iso_ratio', 'signal_min', 'buffer_radius', 'wang_radius', 'pseudo_likelihood', 'target_b_iso_model_scale', 'b_iso', 'b_sharpen', 'resolution_dependent_b', 'normalize_amplitudes_in_resdep', 'region_weight', 'sa_percent', 'n_bins', 'eps', 'max_regions_to_test', 'regions_to_keep', 'fraction_occupied', 'rmsd', 'rmsd_resolution_factor', 'k_sol', 'b_sol', 'fraction_complete', 'nproc', 'multiprocessing', 'queue_run_command', 'verbose', ]: x = var_dict.get(param) if x is not None: if type(x) == type([1, 2, 3]): xx = [] for k in x: xx.append(str(k)) args.append("%s = %s" %(param, " ".join(xx))) else: args.append("%s = %s" %(param, x)) local_params = get_params_from_args(args) # Set solvent content from molecular_mass if present if molecular_mass and not solvent_fraction: solvent_fraction = get_solvent_fraction_from_molecular_mass( crystal_symmetry = crystal_symmetry, molecular_mass = molecular_mass, out = out) # Test to see if we can use adjusted_path_length as target if local_params.map_modification.sharpening_target == \ 'adjusted_path_length': print( "Checking to make sure we can use 'adjusted_path_length'as target...", end = ' ', file = out) try: from phenix.autosol.trace_and_build import trace_and_build except Exception as e: raise Sorry("Please set sharpening target to something other than "+ "adjusted_path_length (not available)") print("OK", file = out) if (local_params.input_files.seq_file or local_params.crystal_info.sequence) and \ not local_params.crystal_info.solvent_content and \ not solvent_fraction: # 2017-12-19 solvent_fraction = get_solvent_fraction(local_params, crystal_symmetry = crystal_symmetry, ncs_copies = ncs_copies, out = out) si.update_with_params(params = local_params, crystal_symmetry = crystal_symmetry, is_crystal = is_crystal, solvent_fraction = solvent_fraction, ncs_copies = ncs_copies, n_residues = n_residues, auto_sharpen = auto_sharpen, sharpening_method = sharpening_method, pdb_inp = pdb_inp, half_map_data_list = half_map_data_list, ) return si def bounds_to_frac(b, map_data): a = map_data.all() return b[0]/a[0], b[1]/a[1], b[2]/a[2] def bounds_to_cart(b, map_data, crystal_symmetry = None): bb = bounds_to_frac(b, map_data) a, b, c = crystal_symmetry.unit_cell().parameters()[:3] return a*bb[0], b*bb[1], c*bb[2] def select_inside_box(lower_bounds = None, upper_bounds = None, xrs = None, hierarchy = None): if not hierarchy or not xrs: return None selection = flex.bool(xrs.scatterers().size()) for atom_group in hierarchy.atom_groups(): for atom in atom_group.atoms(): if atom.xyz[0]>= lower_bounds[0] and \ atom.xyz[0]<= upper_bounds[0] and \ atom.xyz[1]>= lower_bounds[1] and \ atom.xyz[1]<= upper_bounds[1] and \ atom.xyz[2]>= lower_bounds[2] and \ atom.xyz[2]<= upper_bounds[2]: selection[atom.i_seq] = True asc1 = hierarchy.atom_selection_cache() return hierarchy.select(selection) def make_empty_map(template_map = None, value = 0.): # Create empty map original_map_in_box empty_map = flex.double(template_map.as_1d().as_double().size(), value) empty_map.reshape(flex.grid(template_map.all())) return empty_map def sum_box_data(starting_map = None, box_map = None, lower_bounds = None, upper_bounds = None): # sum box data into starting_map #Pull out current starting_map data starting_box_data = maptbx.copy(starting_map, tuple(lower_bounds), tuple(upper_bounds)) assert starting_box_data.all() == box_map.all() # Add to box_map starting_box_data = starting_box_data.as_1d() box_map_data = box_map.as_1d() starting_box_data+= box_map_data # put back into shape starting_box_data.reshape(flex.grid(box_map.all())) maptbx.set_box( map_data_from = starting_box_data, map_data_to = starting_map, start = lower_bounds, end = upper_bounds) return starting_map def copy_box_data(starting_map = None, box_map = None, lower_bounds = None, upper_bounds = None): # Copy box data into original_map_in_box maptbx.set_box( map_data_from = box_map, map_data_to = starting_map, start = lower_bounds, end = upper_bounds) return starting_map def select_box_map_data(si = None, map_data = None, first_half_map_data = None, second_half_map_data = None, pdb_inp = None, get_solvent_fraction = True, # XXX test not doing this... n_min = 30, # at least 30 atoms to run model sharpening restrict_map_size = None, out = sys.stdout, local_out = sys.stdout): box_solvent_fraction = None solvent_fraction = si.solvent_fraction crystal_symmetry = si.crystal_symmetry box_size = si.box_size lower_bounds = None upper_bounds = None smoothed_box_mask_data = None original_box_map_data = None # n_buffer = None if (not pdb_inp and not si.box_in_auto_sharpen) and ( first_half_map_data and second_half_map_data): print("Creating density-based soft mask and applying to half-map data", file = out) if not si.soft_mask: raise Sorry( "Need to set soft_mask = True for half-map sharpening without model") # NOTE: could precede this by density_select on map_data, save bounds and # cut out half-maps with those bounds. That case could # cover si.soft_mask = False half_map_data_list = [first_half_map_data, second_half_map_data] box_mask_data, box_map, half_map_data_list, \ box_solvent_fraction, smoothed_box_mask_data, original_box_map_data = \ get_and_apply_soft_mask_to_maps( resolution = si.resolution, wang_radius = si.wang_radius, buffer_radius = si.buffer_radius, map_data = map_data, crystal_symmetry = crystal_symmetry, half_map_data_list = half_map_data_list, out = out) box_first_half_map, box_second_half_map = half_map_data_list box_crystal_symmetry = crystal_symmetry box_pdb_inp = pdb_inp elif pdb_inp or ( si.density_select_in_auto_sharpen and not si.box_in_auto_sharpen): # use map_box for pdb_inp (mask with model) # also use map_box for density_select_in_auto_sharpen sharpening because # need to use the same density select for all 3 maps. # XXX Perhaps we canuse above method for pdb_inp assert not si.local_sharpening if pdb_inp: print("Using map_box based on input model", file = out) hierarchy = pdb_inp.construct_hierarchy() max_box_fraction = si.max_box_fraction si.density_select_in_auto_sharpen = False else: #print >>out, "Using density_select in map_box" hierarchy = None assert si.density_select_in_auto_sharpen max_box_fraction = si.density_select_max_box_fraction #----------------------trimming model------------------------------- if si.box_center: # Have model but center at box_center and trim hierarchy lower_bounds, upper_bounds = box_from_center(si = si, map_data = map_data, out = out) if si.soft_mask: n_buffer = get_grid_units(map_data = map_data, crystal_symmetry = crystal_symmetry, radius = si.resolution, out = out) n_buffer = int(0.5+n_buffer*1.5) else: n_buffer = 0 lower_bounds, upper_bounds = put_bounds_in_range( lower_bounds = lower_bounds, upper_bounds = upper_bounds, box_size = box_size, n_buffer = n_buffer, n_real = map_data.all(), out = out) lower_frac = bounds_to_frac(lower_bounds, map_data) upper_frac = bounds_to_frac(upper_bounds, map_data) lower_cart = bounds_to_cart( lower_bounds, map_data, crystal_symmetry = crystal_symmetry) upper_cart = bounds_to_cart( upper_bounds, map_data, crystal_symmetry = crystal_symmetry) if hierarchy: # trimming hierarchy to box and then using trimmed # hierarchy in map_box to create actual box xrs = hierarchy.extract_xray_structure( crystal_symmetry = si.crystal_symmetry) # find everything in box sel_hierarchy = select_inside_box(lower_bounds = lower_cart, upper_bounds = upper_cart, xrs = xrs, hierarchy = hierarchy) n = sel_hierarchy.overall_counts().n_atoms print("Selected atoms inside box: %d" %(n), file = out) if n>out, "Using density_select in map_box" if si.density_select_threshold_in_auto_sharpen is not None: args.append('density_select_threshold = %s' %( si.density_select_threshold_in_auto_sharpen)) elif si.box_in_auto_sharpen and not si.mask_atoms: print("Using map_box with model", file = out) elif si.mask_atoms: print("Using map_box with model and mask_atoms", file = out) args.append('mask_atoms = True') if si.mask_atoms_atom_radius: args.append('mask_atoms_atom_radius = %s' %(si.mask_atoms_atom_radius)) if si.value_outside_atoms: args.append('value_outside_atoms = %s' %(si.value_outside_atoms)) if si.soft_mask: print("Using soft mask", file = out) args.append('soft_mask = %s' %(si.soft_mask)) args.append('soft_mask_radius = %s' %(si.resolution)) else: raise Sorry("Unknown choice in select_box_data") if restrict_map_size: args.append('restrict_map_size = True') print("Getting map as box now", file = out) local_hierarchy = None if hierarchy: local_hierarchy = hierarchy.deep_copy() # run_map_box modifies its argument assert isinstance(si.wrapping, bool) # wrapping must be defined box = run_map_box(args, map_data = map_data, pdb_hierarchy = local_hierarchy, write_output_files = False, wrapping = si.wrapping, crystal_symmetry = crystal_symmetry, log = out) lower_bounds = box.gridding_first upper_bounds = box.gridding_last box_map = box.map_box box_map = scale_map(box_map, out = out) box_crystal_symmetry = box.box_crystal_symmetry if box.hierarchy: box_pdb_inp = box.hierarchy.as_pdb_input() else: box_pdb_inp = None if first_half_map_data: print("Getting first map as box", file = out) if hierarchy: local_hierarchy = hierarchy.deep_copy() # required box_first = run_map_box(args, map_data = first_half_map_data, pdb_hierarchy = local_hierarchy, write_output_files = False, lower_bounds = lower_bounds, upper_bounds = upper_bounds, crystal_symmetry = crystal_symmetry, wrapping = si.wrapping, log = out) box_first_half_map = box_first.map_box.as_double() else: box_first_half_map = None if second_half_map_data: print("Getting second map as box", file = out) if hierarchy: local_hierarchy = hierarchy.deep_copy() # required box_second = run_map_box(args, map_data = second_half_map_data, pdb_hierarchy = local_hierarchy, write_output_files = False, lower_bounds = lower_bounds, upper_bounds = upper_bounds, crystal_symmetry = crystal_symmetry, wrapping = si.wrapping, log = out) box_second_half_map = box_second.map_box.as_double() else: box_second_half_map = None else: # cut out box based on box_center or regions if si.box_center: # center at box_center print("Cutting out box based on center at (%.2f, %.2f, %.2f) " %( si.box_center), file = out) lower_bounds, upper_bounds = box_from_center(si = si, map_data = map_data, out = out) elif si.use_weak_density: print("Cutting out box based on centering on weak density", file = out) lower_bounds, upper_bounds = box_of_smallest_region(si = si, map_data = map_data, out = out) else: print("Cutting out box based on centering on strong density", file = out) lower_bounds, upper_bounds = box_of_biggest_region(si = si, map_data = map_data, out = out) if si.soft_mask: n_buffer = get_grid_units(map_data = map_data, crystal_symmetry = crystal_symmetry, radius = si.resolution, out = out) n_buffer = int(0.5+n_buffer*1.5) else: n_buffer = 0 lower_bounds, upper_bounds = put_bounds_in_range( lower_bounds = lower_bounds, upper_bounds = upper_bounds, box_size = box_size, n_buffer = n_buffer, n_real = map_data.all(), out = out) # select map data inside this box print("\nSelecting map data inside box", file = out) box_map, box_crystal_symmetry, \ smoothed_box_mask_data, original_box_map_data = cut_out_map( map_data = map_data.as_double(), crystal_symmetry = crystal_symmetry, soft_mask = si.soft_mask, soft_mask_radius = si.resolution, resolution = si.resolution, shift_origin = True, min_point = lower_bounds, max_point = upper_bounds, out = out) box_pdb_inp = None if first_half_map_data: box_first_half_map, box_first_crystal_symmetry, \ dummy_smoothed_box_mask_data, dummy_original_box_map_data = cut_out_map( map_data = first_half_map_data.as_double(), crystal_symmetry = crystal_symmetry, soft_mask = si.soft_mask, soft_mask_radius = si.resolution, resolution = si.resolution, shift_origin = True, min_point = lower_bounds, max_point = upper_bounds, out = local_out) else: box_first_half_map = None if second_half_map_data: box_second_half_map, box_second_crystal_symmetry, \ dummy_smoothed_box_mask_data, dummy_original_box_map_data = cut_out_map( map_data = second_half_map_data.as_double(), crystal_symmetry = crystal_symmetry, soft_mask = si.soft_mask, soft_mask_radius = si.resolution, resolution = si.resolution, shift_origin = True, min_point = lower_bounds, max_point = upper_bounds, out = local_out) else: box_second_half_map = None if not box_map or ( (not pdb_inp and not second_half_map_data) and \ box_map.size() > si.max_box_fraction* map_data.size()): return None, map_data, first_half_map_data, \ second_half_map_data, crystal_symmetry, None, \ smoothed_box_mask_data, None, None # no point # figure out solvent fraction in this box... if get_solvent_fraction: # if box_solvent_fraction is None: box_solvent_fraction = get_iterated_solvent_fraction( crystal_symmetry = box_crystal_symmetry, map = box_map, fraction_of_max_mask_threshold = si.fraction_of_max_mask_threshold, mask_resolution = si.resolution, out = out) if box_solvent_fraction is None: box_solvent_fraction = si.solvent_fraction print("Using overall solvent fraction for box", file = out) print("Local solvent fraction: %7.2f" %(box_solvent_fraction), file = out) else: box_solvent_fraction = None box_sharpening_info_obj = box_sharpening_info( lower_bounds = lower_bounds, upper_bounds = upper_bounds, n_real = box_map.all(), scale_max = si.scale_max, wrapping = False, crystal_symmetry = box_crystal_symmetry, solvent_fraction = box_solvent_fraction) return box_pdb_inp, box_map, box_first_half_map, box_second_half_map, \ box_crystal_symmetry, box_sharpening_info_obj, \ smoothed_box_mask_data, original_box_map_data, n_buffer def inside_zero_one(xyz): from scitbx.matrix import col offset = xyz-col((0.5, 0.5, 0.5)) lower_int = offset.iround().as_vec3_double() return xyz-lower_int def move_xyz_inside_cell(xyz_cart = None, crystal_symmetry = None): xyz_local = flex.vec3_double() if type(xyz_cart) == type(xyz_local): xyz_local = xyz_cart is_single = False else: is_single = True xyz_local.append(xyz_cart) xyz_frac = crystal_symmetry.unit_cell().fractionalize(xyz_local) new_xyz_frac = inside_zero_one(xyz_frac) new_xyz_cart = crystal_symmetry.unit_cell().orthogonalize(new_xyz_frac) if is_single: return new_xyz_cart[0] else: return new_xyz_cart def box_from_center( si = None, map_data = None, out = sys.stdout): cx, cy, cz = si.crystal_symmetry.unit_cell().fractionalize(si.box_center) if cx<0 or cx>1 or cy<0 or cy>1 or cz<0 or cz>1: print("Moving box center inside (0, 1)", file = out) si.box_center = move_xyz_inside_cell( xyz_cart = si.box_center, crystal_symmetry = si.crystal_symmetry) cx, cy, cz = si.crystal_symmetry.unit_cell().fractionalize(si.box_center) print("\nBox centered at (%7.2f, %7.2f, %7.2f) A" %( tuple(si.box_center)), file = out) ax, ay, az = map_data.all() cgx, cgy, cgz = int(0.5+ax*cx), int(0.5+ay*cy), int(0.5+az*cz), print("Box grid centered at (%d, %d, %d)\n" %(cgx, cgy, cgz), file = out) return (cgx, cgy, cgz), (cgx, cgy, cgz) def box_of_smallest_region(si = None, map_data = None, return_as_list = None, out = sys.stdout): return box_of_biggest_region(si = si, map_data = map_data, return_as_list = return_as_list, use_smallest = True, out = out) def box_of_biggest_region(si = None, map_data = None, return_as_list = None, use_smallest = False, out = sys.stdout): n_residues = si.n_residues ncs_copies = si.ncs_copies solvent_fraction = si.solvent_fraction b_vs_region = b_vs_region_info() co, sorted_by_volume, min_b, max_b, unique_expected_regions, best_score, \ new_threshold, starting_density_threshold = \ get_connectivity( b_vs_region = b_vs_region, map_data = map_data, n_residues = n_residues, ncs_copies = ncs_copies, solvent_fraction = solvent_fraction, min_volume = si.min_volume, min_ratio = si.min_ratio, fraction_occupied = si.fraction_occupied, wrapping = si.wrapping, residues_per_region = si.residues_per_region, max_ratio_to_target = si.max_ratio_to_target, min_ratio_to_target = si.min_ratio_to_target, min_ratio_of_ncs_copy_to_first = si.min_ratio_of_ncs_copy_to_first, starting_density_threshold = si.starting_density_threshold, density_threshold = si.density_threshold, crystal_symmetry = si.crystal_symmetry, chain_type = si.chain_type, verbose = si.verbose, out = out) if len(sorted_by_volume)<2: return # nothing to do if use_smallest: small_ratio = 0.25 maximum_position_ratio = 0.75 v1, i1 = sorted_by_volume[1] v_small = small_ratio*v1 maximum_position_small = maximum_position_ratio*(len(sorted_by_volume)-1)+1 best_pos = 1 ii = 0 for v, i in sorted_by_volume[1:]: ii+= 1 if v < v_small: continue if ii > maximum_position_small: continue best_pos = ii v, i = sorted_by_volume[best_pos] print("\nVolume of target region %d: %d grid points: "%(best_pos, v), file = out) else: # usual v, i = sorted_by_volume[1] print("\nVolume of largest region: %d grid points: "%(v), file = out) print("Region %3d (%3d) volume:%5d X:%6d - %6d Y:%6d - %6d Z:%6d - %6d "%( 1, i, v, min_b[i][0], max_b[i][0], min_b[i][1], max_b[i][1], min_b[i][2], max_b[i][2]), file = out) if (not return_as_list): return min_b[i], max_b[i] else: # return a list of centers centers_frac = flex.vec3_double() a1, a2, a3 = map_data.all() for v, i in sorted_by_volume[1:]: centers_frac.append( tuple(( (min_b[i][0]+max_b[i][0])/(2.*a1), (min_b[i][1]+max_b[i][1])/(2.*a2), (min_b[i][2]+max_b[i][2])/(2.*a3), )) ) return centers_frac def get_fft_map(n_real = None, map_coeffs = None): from cctbx import maptbx from cctbx.maptbx import crystal_gridding if n_real: cg = crystal_gridding( unit_cell = map_coeffs.crystal_symmetry().unit_cell(), space_group_info = map_coeffs.crystal_symmetry().space_group_info(), pre_determined_n_real = n_real) else: cg = None ccs = map_coeffs.crystal_symmetry() fft_map = map_coeffs.fft_map( resolution_factor = 0.25, crystal_gridding = cg, symmetry_flags = maptbx.use_space_group_symmetry) fft_map.apply_sigma_scaling() return fft_map def average_from_bounds(lower, upper, grid_all = None): avg = [] for u, l in zip(upper, lower): avg.append(0.5*(u+l)) if grid_all: avg_fract = [] for a, g in zip(avg, grid_all): avg_fract.append(a/g) avg = avg_fract return avg def get_ncs_copies(site_cart, ncs_object = None, only_inside_box = None, unit_cell = None): ncs_group = ncs_object.ncs_groups()[0] from scitbx.array_family import flex from scitbx.matrix import col sites_cart_ncs = flex.vec3_double() for t, r in zip(ncs_group.translations_orth_inv(), ncs_group.rota_matrices_inv()): sites_cart_ncs.append(r * col(site_cart) + t) if only_inside_box: assert unit_cell is not None sites_frac_ncs = unit_cell.fractionalize(sites_cart_ncs) new_sites_frac = flex.vec3_double() for x in sites_frac_ncs: if x[0]>= 0 and x[0]<= 1 and \ x[1]>= 0 and x[1]<= 1 and \ x[2]>= 0 and x[2]<= 1: new_sites_frac.append(x) sites_cart_ncs = unit_cell.orthogonalize(new_sites_frac) return sites_cart_ncs def fit_bounds_inside_box(lower, upper, box_size = None, all = None): # adjust bounds so upper>lower and box size is at least box_size new_lower = [] new_upper = [] if box_size: for u, l, s, a in zip(upper, lower, box_size, all): ss = u-l+1 delta = int((1+s-ss)/2) # desired increase in size, to subtract from l l = max(0, l-delta) ss = u-l+1 delta = (s-ss) # desired increase in size, to add to u u = min(a-1, u+delta) l = max(0, l) new_lower.append(l) new_upper.append(u) else: for u, l, a in zip(upper, lower, all): u = min(a-1, u) l = max(0, l) new_lower.append(l) new_upper.append(u) return new_lower, new_upper def split_boxes(lower = None, upper = None, target_size = None, target_n_overlap = None, fix_target_size_and_overlap = None): # purpose: split the region from lower to upper into overlapping # boxes of size about target_size # NOTE: does not actually use target_n_overlap unless # fix_target_size_and_overlap is set all_box_list = [] for l, u, ts in zip (lower, upper, target_size): n = u+1-l # offset defined by: ts-offset + ts-offset+...+ts = n # ts*n_box-offset*(n_box-1) = n approx # n_box (ts-offset) +offset = n # n_box = (n-offset)/(ts-offset) assert ts>target_n_overlap if fix_target_size_and_overlap: n_box = (n-target_n_overlap-1)/(ts-target_n_overlap) if n_box>int(n_box): n_box = int(n_box)+1 else: n_box = int(n_box) new_target_size = ts offset = ts-target_n_overlap else: # original version n_box = max(1, (n+(3*ts//4))//ts) new_target_size = int(0.9+n/n_box) offset = int(0.9+(n-new_target_size)/max(1, n_box-1)) box_list = [] for i in range(n_box): start_pos = max(l, l+i*offset) end_pos = min(u, start_pos+new_target_size) if fix_target_size_and_overlap: start_pos = max(0, min(start_pos, end_pos-new_target_size)) box_list.append([start_pos, end_pos]) all_box_list.append(box_list) new_lower_upper_list = [] for xs, xe in all_box_list[0]: for ys, ye in all_box_list[1]: for zs, ze in all_box_list[2]: new_lower_upper_list.append([(xs, ys, zs, ), (xe, ye, ze, )]) return new_lower_upper_list def get_target_boxes(si = None, ncs_obj = None, map = None, pdb_inp = None, out = sys.stdout): print(80*"-", file = out) print("Getting segmented map to ID locations for sharpening", file = out) print(80*"-", file = out) if si.input_weight_map_pickle_file: from libtbx import easy_pickle file_name = si.input_weight_map_pickle_file print("Loading segmentation data from %s" %(file_name), file = out) tracking_data = easy_pickle.load(file_name) else: args = [ 'resolution = %s' %(si.resolution), 'seq_file = %s' %(si.seq_file), 'sequence = %s' %(si.sequence), 'solvent_content = %s' %(si.solvent_fraction), 'auto_sharpen = False', # XXX could sharpen overall 'write_output_maps = True', 'add_neighbors = False', 'density_select = False', ] if si.is_crystal: args.append("is_crystal = True") ncs_group_obj, remainder_ncs_group_obj, tracking_data = run( args, map_data = map.deep_copy(), ncs_obj = ncs_obj, crystal_symmetry = si.crystal_symmetry) if si.output_weight_map_pickle_file: from libtbx import easy_pickle file_name = os.path.join(si.output_directory, si.output_weight_map_pickle_file) print("Dumping segmentation data to %s" %(file_name), file = out) easy_pickle.dump(file_name, tracking_data) if not ncs_obj or ncs_obj.max_operators() == 0: from mmtbx.ncs.ncs import ncs ncs_obj = ncs() ncs_obj.set_unit_ncs() print("Regions in this map:", file = out) centers_frac = flex.vec3_double() upper_bounds_list = [] lower_bounds_list = [] if pdb_inp and pdb_inp.atoms().extract_xyz().size()>1: xyz_list = pdb_inp.atoms().extract_xyz() i_end = xyz_list.size() n_centers = min(i_end, max(1, len(tracking_data.output_region_map_info_list))) n_steps = min(n_centers, xyz_list.size()) i_step = int(0.5+min(i_end/2, i_end/n_steps)) # about n_centers but up to n_atoms i_start = max(1, int(0.5+i_step/2)) from scitbx.matrix import col ma = map.all() for i in range(i_start, i_end, i_step): lower_cart = col(xyz_list[i]) lower_frac = si.crystal_symmetry.unit_cell().fractionalize(lower_cart) lower = [ int(0.5+ma[0]*lower_frac[0]), int(0.5+ma[1]*lower_frac[1]), int(0.5+ma[2]*lower_frac[2])] lower, upper = fit_bounds_inside_box( lower, lower, box_size = si.box_size, all = map.all()) upper_bounds_list.append(upper) lower_bounds_list.append(lower) average_fract = average_from_bounds(lower, upper, grid_all = map.all()) centers_frac.append(average_fract) else: for map_info_obj in tracking_data.output_region_map_info_list: lower, upper = map_info_obj.lower_upper_bounds() lower, upper = fit_bounds_inside_box( lower, upper, box_size = None, # take the whole region, not just center all = map.all()) for lower, upper in split_boxes(lower = lower, upper = upper, target_size = si.box_size, target_n_overlap = si.target_n_overlap): upper_bounds_list.append(upper) lower_bounds_list.append(lower) average_fract = average_from_bounds(lower, upper, grid_all = map.all()) centers_frac.append(average_fract) centers_cart = si.crystal_symmetry.unit_cell().orthogonalize(centers_frac) # Make ncs-related centers print("NCS ops:", ncs_obj.max_operators(), file = out) centers_cart_ncs_list = [] for i in range(centers_cart.size()): centers_cart_ncs_list.append(get_ncs_copies( centers_cart[i], ncs_object = ncs_obj, only_inside_box = True, unit_cell = si.crystal_symmetry.unit_cell()) ) all_cart = flex.vec3_double() for center_list in centers_cart_ncs_list: all_cart.extend(center_list) sharpening_centers_file = os.path.join( si.output_directory, "sharpening_centers.pdb") write_atoms(file_name = sharpening_centers_file, crystal_symmetry = si.crystal_symmetry, sites = centers_cart) ncs_sharpening_centers_file = os.path.join( si.output_directory, "ncs_sharpening_centers.pdb") write_atoms(file_name = ncs_sharpening_centers_file, crystal_symmetry = si.crystal_symmetry, sites = all_cart) print("\nSharpening centers (matching shifted_map_file).\n\n "+\ "Written to: \n%s \n%s\n"%( sharpening_centers_file, ncs_sharpening_centers_file), file = out) for i in range(centers_cart.size()): print("Center: %s (%7.2f, %7.2f, %7.2f) Bounds: %s :: %s " %( i, centers_cart[i][0], centers_cart[i][1], centers_cart[i][2], str(lower_bounds_list[i]), str(upper_bounds_list[i])), file = out) print(80*"-", file = out) print("Done getting segmented map to ID locations for sharpening", file = out) print(80*"-", file = out) return upper_bounds_list, lower_bounds_list, \ centers_cart_ncs_list, centers_cart, all_cart def get_box_size(lower_bound = None, upper_bound = None): box_size = [] for lb, ub in zip(lower_bound, upper_bound): box_size.append(ub-lb+1) return box_size def mean_dist_to_nearest_neighbor(all_cart): if all_cart.size()<2: # nothing to check return None sum_dist = 0. sum_n = 0. for i in range(all_cart.size()): xyz = all_cart[i:i+1] others = all_cart[:i] others.extend(all_cart[i+1:]) sum_dist+= get_closest_dist(xyz, others) sum_n+= 1. return sum_dist/max(1., sum_n) def run_local_sharpening(si = None, auto_sharpen_methods = None, map = None, ncs_obj = None, half_map_data_list = None, pdb_inp = None, out = sys.stdout): print(80*"-", file = out) print("Running local sharpening", file = out) print(80*"-", file = out) # run auto_sharpen_map_or_map_coeffs with box_in_auto_sharpen = True and # centered at different places. Identify the places as centers of regions. # Run on au of NCS and apply NCS to get remaining positions if si.overall_before_local: # first do overall sharpening of the map to get it about right print(80*"*", file = out) print("\nSharpening map overall before carrying out local sharpening\n", file = out) overall_si = deepcopy(si) overall_si.local_sharpening = False # don't local sharpen overall_si = auto_sharpen_map_or_map_coeffs(si = overall_si, auto_sharpen_methods = auto_sharpen_methods, map = map, half_map_data_list = half_map_data_list, pdb_inp = pdb_inp, out = out) sharpened_map = overall_si.map_data print("\nDone sharpening map overall before carrying out local sharpening\n", file = out) print(80*"*", file = out) else: sharpened_map = map # Accumulate sums starting_weight = 0.01 # put starting map everywhere with low weight sum_weight_map = make_empty_map(template_map = map, value = starting_weight) # in case a pixel is not covered... sum_weight_value_map = starting_weight*sharpened_map.deep_copy() print("\nUsing overall map for any regions where "+\ "no local information is present", file = out) id_list = [] b_iso_list = flex.double() starting_b_iso_list = flex.double() # use sharpened map here upper_bounds_list, lower_bounds_list, \ centers_cart_ncs_list, centers_cart, all_cart = \ get_target_boxes(si = si, map = sharpened_map, ncs_obj = ncs_obj, pdb_inp = pdb_inp, out = out) dist = mean_dist_to_nearest_neighbor(all_cart) if not dist: dist = 10. if not si.smoothing_radius: print("No nearest neighbors...best to set smoothing radius", file = out) print("\nMean distance to nearest center is %7.2f A " %( dist), file = out) if not si.smoothing_radius: si.smoothing_radius = float("%.0f" %(dist*2/3)) # 10A from nearest neighbor print("Using %s A for smoothing radius" %(si.smoothing_radius), file = out) i = -1 for ub, lb, centers_ncs_cart, center_cart in zip( upper_bounds_list, lower_bounds_list, centers_cart_ncs_list, centers_cart): i+= 1 if si.select_sharpened_map is not None and i != si.select_sharpened_map: continue map_file_name = 'sharpened_map_%s.ccp4' %(i) if 0 and si.read_sharpened_maps: # cannot do this as no bounds print("\nReading sharpened map directly from %s" %(map_file_name), file = out) result = get_map_object(file_name = map_file_name, out = out) local_map_data = result[0] else: local_si = deepcopy(si) local_si.local_sharpening = False # don't do it again local_si.box_size = get_box_size(lower_bound = lb, upper_bound = ub) local_si.box_center = center_cart local_si.box_in_auto_sharpen = True local_si.density_select_in_auto_sharpen = False local_si.use_local_aniso = si.local_aniso_in_local_sharpening local_si.remove_aniso = si.local_aniso_in_local_sharpening local_si.max_box_fraction = 999 # just bigger than 1 local_si.density_select_max_box_fraction = 999 local_si.nproc = 1 print(80*"+", file = out) print("Getting local sharpening for box %s" %(i), file = out) print(80*"+", file = out) bsi = auto_sharpen_map_or_map_coeffs(si = local_si, auto_sharpen_methods = auto_sharpen_methods, map = sharpened_map, half_map_data_list = half_map_data_list, pdb_inp = pdb_inp, return_bsi = True, # just return the bsi of sharpened data out = out) if not bsi.map_data: print("\nNo result for local map %s ...skipping" %(i), file = out) continue # merge with background using bsi.smoothed_box_mask_data if bsi.smoothed_box_mask_data: print("Merging small map into overall map in soft-mask region", file = out) bsi.merge_into_overall_map(overall_map = map) # XXX overall_map not used # Now remove buffer region if bsi.n_buffer: # extract just the good part print("Removing buffer from small map", file = out) bsi.remove_buffer(out = out) weight_data = bsi.get_gaussian_weighting(out = out) weighted_data = bsi.map_data*weight_data sum_weight_value_map = sum_box_data(starting_map = sum_weight_value_map, box_map = weighted_data, lower_bounds = bsi.lower_bounds, upper_bounds = bsi.upper_bounds) sum_weight_map = sum_box_data(starting_map = sum_weight_map, box_map = weight_data, lower_bounds = bsi.lower_bounds, upper_bounds = bsi.upper_bounds) id_list.append(i) starting_b_iso_list.append(bsi.starting_b_iso) b_iso_list.append(bsi.b_iso) print(80*"+", file = out) print("End of getting local sharpening for small box %s" %(i), file = out) print(80*"+", file = out) print("\nOverall map created from total of %s local maps" %(i), file = out) if si.overall_before_local: print("Note: overall map already sharpened with global sharpening", file = out) if starting_b_iso_list.size()<1: print("No results for local sharpening...", file = out) else: print("Summary of b_iso values by local map:", file = out) print(" ID Starting B-iso Sharpened B-iso", file = out) for i, starting_b_iso, b_iso in zip(id_list, starting_b_iso_list, b_iso_list): print(" %4s %7.2f %7.2f" %(i, starting_b_iso, b_iso), file = out) print("\nMean %7.2f %7.2f" %( starting_b_iso_list.min_max_mean().mean, b_iso_list.min_max_mean().mean), file = out) si.map_data = sum_weight_value_map/sum_weight_map # Get overall b_iso... print("\nGetting overall b_iso of composite map...", file = out) map_coeffs_aa, map_coeffs, f_array, phases = effective_b_iso( map_data = si.map_data, resolution = si.resolution, d_min_ratio = si.d_min_ratio, scale_max = si.scale_max, crystal_symmetry = si.crystal_symmetry, out = out) print(80*"+", file = out) print("End of getting local sharpening ", file = out) print(80*"+", file = out) return si def auto_sharpen_map_or_map_coeffs( si = None, resolution = None, # resolution is required crystal_symmetry = None, # supply crystal_symmetry and map or map = None, # map and n_real wrapping = None, half_map_data_list = None, # two half-maps matching map is_crystal = None, map_coeffs = None, pdb_inp = None, ncs_obj = None, seq_file = None, sequence = None, rmsd = None, rmsd_resolution_factor = None, k_sol = None, b_sol = None, fraction_complete = None, n_real = None, solvent_content = None, molecular_mass = None, region_weight = None, sa_percent = None, n_bins = None, eps = None, max_regions_to_test = None, regions_to_keep = None, fraction_occupied = None, input_weight_map_pickle_file = None, output_weight_map_pickle_file = None, read_sharpened_maps = None, write_sharpened_maps = None, select_sharpened_map = None, output_directory = None, smoothing_radius = None, local_sharpening = None, local_aniso_in_local_sharpening = None, overall_before_local = None, use_local_aniso = None, auto_sharpen = None, box_in_auto_sharpen = None, # n_residues, ncs_copies required if not False density_select_in_auto_sharpen = None, density_select_threshold_in_auto_sharpen = None, allow_box_if_b_iso_set = None, use_weak_density = None, discard_if_worse = None, n_residues = None, ncs_copies = None, box_center = None, remove_aniso = None, box_size = None, target_n_overlap = None, lower_bounds = None, upper_bounds = None, restrict_map_size = None, auto_sharpen_methods = None, residual_target = None, sharpening_target = None, d_min_ratio = None, scale_max = None, input_d_cut = None, b_blur_hires = None, max_box_fraction = None, cc_cut = None, max_cc_for_rescale = None, scale_using_last = None, density_select_max_box_fraction = None, mask_atoms = None, mask_atoms_atom_radius = None, value_outside_atoms = None, soft_mask = None, tol_r = None, abs_tol_t = None, rel_tol_t = None, require_helical_or_point_group_symmetry = None, max_helical_operators = None, k_sharpen = None, optimize_d_cut = None, optimize_b_blur_hires = None, iterate = None, search_b_min = None, search_b_max = None, search_b_n = None, adjust_region_weight = None, region_weight_method = None, region_weight_factor = None, region_weight_buffer = None, region_weight_default = None, target_b_iso_ratio = None, signal_min = None, buffer_radius = None, wang_radius = None, pseudo_likelihood = None, target_b_iso_model_scale = None, b_iso = None, # if set, use it b_sharpen = None, # if set, use it resolution_dependent_b = None, # if set, use it normalize_amplitudes_in_resdep = None, # if set, use it return_bsi = False, verbose = None, resolve_size = None, nproc = None, multiprocessing = None, queue_run_command = None, out = sys.stdout): if si: # resolution = si.resolution crystal_symmetry = si.crystal_symmetry if not auto_sharpen: auto_sharpen = si.auto_sharpen if verbose is None: verbose = si.verbose if resolve_size is None: resolve_size = si.resolve_size if auto_sharpen is None: auto_sharpen = True if map_coeffs and not resolution: resolution = map_coeffs.d_min() if map_coeffs and not crystal_symmetry: crystal_symmetry = map_coeffs.crystal_symmetry() assert resolution is not None if map: return_as_map = True else: # convert from structure factors to create map if necessary map = get_fft_map(n_real = n_real, map_coeffs = map_coeffs).real_map_unpadded() return_as_map = False # Set ncs_copies if possible if ncs_copies is None and ncs_obj and ncs_obj.max_operators(): ncs_copies = ncs_obj.max_operators() print("Set ncs copies based on ncs_obj to %s" %(ncs_copies), file = out) # Determine if we are running model_sharpening if half_map_data_list and len(half_map_data_list) == 2: if auto_sharpen_methods != ['external_map_sharpening']: auto_sharpen_methods = ['half_map_sharpening'] elif pdb_inp: auto_sharpen_methods = ['model_sharpening'] if not si: # Copy parameters to si (sharpening_info_object) si = set_up_si(var_dict = locals(), crystal_symmetry = crystal_symmetry, is_crystal = is_crystal, solvent_fraction = solvent_content, molecular_mass = molecular_mass, auto_sharpen = auto_sharpen, map = map, verbose = verbose, half_map_data_list = half_map_data_list, pdb_inp = pdb_inp, ncs_copies = ncs_copies, n_residues = n_residues, out = out) if wrapping is not None: si.wrapping = wrapping # Figure out solvent fraction if si.solvent_fraction is None: si.solvent_fraction = get_iterated_solvent_fraction( crystal_symmetry = crystal_symmetry, verbose = si.verbose, resolve_size = si.resolve_size, fraction_of_max_mask_threshold = si.fraction_of_max_mask_threshold, mask_resolution = si.resolution, map = map, out = out) if si.solvent_fraction: print("Estimated solvent content: %.2f" %(si.solvent_fraction), file = out) else: raise Sorry("Unable to estimate solvent content...please supply "+ "solvent_content \nor molecular_mass") # Determine if we are running half-map or model_sharpening if half_map_data_list and len(half_map_data_list) == 2: first_half_map_data = half_map_data_list[0] second_half_map_data = half_map_data_list[1] else: first_half_map_data = None second_half_map_data = None # Decide if we are running local sharpening (overlapping set of sharpening # runs at various locations) libtbx.call_back(message = 'sharpen', data = None) if si.local_sharpening: return run_local_sharpening(si = si, auto_sharpen_methods = auto_sharpen_methods, map = map, ncs_obj = ncs_obj, half_map_data_list = half_map_data_list, pdb_inp = pdb_inp, out = out) # Get preliminary values of sharpening working_map = map # use another name for map XXX if si.iterate and not si.preliminary_sharpening_done: si.preliminary_sharpening_done = True si.iterate = False # first do overall sharpening of the map to get it about right print(80*"*", file = out) print("\nSharpening map overall before carrying out final sharpening\n", file = out) overall_si = deepcopy(si) overall_si.local_sharpening = False # don't local sharpen overall_si = auto_sharpen_map_or_map_coeffs(si = overall_si, auto_sharpen_methods = auto_sharpen_methods, map = map, half_map_data_list = half_map_data_list, pdb_inp = pdb_inp, out = out) working_map = overall_si.map_data # Get solvent content again overall_si.solvent_content = None overall_si.solvent_fraction = get_iterated_solvent_fraction( crystal_symmetry = crystal_symmetry, verbose = overall_si.verbose, resolve_size = overall_si.resolve_size, fraction_of_max_mask_threshold = si.fraction_of_max_mask_threshold, mask_resolution = si.resolution, map = working_map, out = out) print("Resetting solvent fraction to %.2f " %( overall_si.solvent_fraction), file = out) si.solvent_fraction = overall_si.solvent_fraction print("\nDone sharpening map overall before carrying out final sharpening\n", file = out) print(80*"*", file = out) si.b_blur_hires = 0. # from now on, don't apply extra blurring else: working_map = map # Now identify optimal sharpening params print(80*"=", file = out) print("\nRunning auto_sharpen to get sharpening parameters\n", file = out) print(80*"=", file = out) result = run_auto_sharpen( # get sharpening parameters standard run si = si, map_data = working_map, first_half_map_data = first_half_map_data, second_half_map_data = second_half_map_data, pdb_inp = pdb_inp, auto_sharpen_methods = auto_sharpen_methods, print_result = False, return_bsi = return_bsi, out = out) if return_bsi: return result # it is a box_sharpening_info object else: si = result print(80*"=", file = out) print("\nDone running auto_sharpen to get sharpening parameters\n", file = out) print(80*"=", file = out) # Apply the optimal sharpening values and save map in si.map_data # First test without sharpening if sharpening_method is b_iso, b and # b_iso is not set if si.sharpening_method in [ 'b_iso', 'b_iso_to_d_cut', 'resolution_dependent'] and b_iso is None: local_si = deepcopy(si) local_si.sharpening_method = 'no_sharpening' local_si.sharpen_and_score_map(map_data = working_map, out = null_out()) print("\nScore for no sharpening: %7.2f " %(local_si.score), file = out) else: local_si = None print(80*"=", file = out) print("\nApplying final sharpening to entire map", file = out) print(80*"=", file = out) si.sharpen_and_score_map(map_data = working_map, set_b_iso = True, out = out) if si.discard_if_worse and local_si and local_si.score > si.score: print("Sharpening did not improve map "+\ "(%7.2f sharpened, %7.2f unsharpened). Discarding sharpened map" %( si.score, local_si.score), file = out) print("\nUse discard_if_worse = False to keep the sharpening", file = out) local_si.sharpen_and_score_map(map_data = working_map, out = out) si = local_si if not si.is_model_sharpening() and not si.is_half_map_sharpening(): si.show_score(out = out) si.show_summary(out = out) return si # si.map_data is the sharpened map def estimate_signal_to_noise(value_list = None, minimum_value_to_include = 0): # get "noise" from rms value of value_list(n) compared with average of n-2, n-1, n+1, n+2 # assumes middle is the high part of the very smooth curve. # Don't include values < minimum_value_to_include mean_square_diff = 0. mean_square_diff_n = 0. for b2, b1, value, p1, p2 in zip(value_list, value_list[1:], value_list[2:], value_list[3:], value_list[4:]): too_low = False for xx in [b2, b1, value, p1, p2]: if xx 0: min_adj_sa = max(value_list[0], value_list[-1]) max_adj_sa = value_list.min_max_mean().max signal_to_noise = (max_adj_sa-min_adj_sa)/max(1.e-10, rmsd) else: signal_to_noise = 0. return signal_to_noise def optimize_b_blur_or_d_cut_or_b_iso( optimization_target = 'b_blur_hires', local_best_si = None, local_best_map_and_b = None, si_id_list = None, si_score_list = None, delta_b = None, original_b_iso = None, f_array = None, phases = None, delta_b_blur_hires = 100, delta_d_cut = 0.25, n_cycle_optimize = 5, min_cycles = 2, n_range = 5, out = sys.stdout): assert optimization_target in ['b_blur_hires', 'd_cut', 'b_iso'] if optimization_target == 'b_blur_hires': print("\nOptimizing b_blur_hires. ", file = out) elif optimization_target == 'd_cut': print("\nOptimizing d_cut. ", file = out) local_best_si.input_d_cut = local_best_si.get_d_cut() elif optimization_target == 'b_iso': print("\nOptimizing b_iso. ", file = out) local_best_si.show_summary(out = out) print("Current best score = %7.3f b_iso = %5.1f b_blur_hires = %5.1f d_cut = %5.1f" %( local_best_si.score, local_best_si.b_iso, local_best_si.b_blur_hires, local_best_si.get_d_cut()), file = out) # existing values: value_dict = {} for id, score in zip(si_id_list, si_score_list): value_dict[id] = score best_score = local_best_si.score delta_b_iso = delta_b local_best_score = best_score improving = True working_best_si = deepcopy(local_best_si) for cycle in range(n_cycle_optimize): if not improving: break print("Optimization cycle %s" %(cycle), file = out) print("Current best score = %7.3f b_iso = %5.1f b_blur_hires = %5.1f d_cut = %5.1f" %( working_best_si.score, working_best_si.b_iso, working_best_si.b_blur_hires, working_best_si.get_d_cut()), file = out) if working_best_si.verbose: print(" B-sharpen B-iso B-blur Adj-SA "+\ "Kurtosis SA-ratio Regions d_cut b_blur_hires", file = out) local_best_working_si = deepcopy(working_best_si) improving = False for jj in range(-n_range, n_range+1): if optimization_target == 'b_blur_hires': # try optimizing b_blur_hires test_b_blur_hires = max(0., working_best_si.b_blur_hires+jj*delta_b_blur_hires) test_d_cut = working_best_si.get_d_cut() test_b_iso = working_best_si.b_iso elif optimization_target == 'd_cut': test_b_blur_hires = working_best_si.b_blur_hires test_d_cut = working_best_si.get_d_cut()+jj*delta_d_cut test_b_iso = working_best_si.b_iso elif optimization_target == 'b_iso': test_b_blur_hires = working_best_si.b_blur_hires test_d_cut = working_best_si.get_d_cut() test_b_iso = working_best_si.b_iso+jj*delta_b_iso id = "%.3f_%.3f_%.3f" %(test_b_iso, test_b_blur_hires, test_d_cut) if id in value_dict: score = value_dict[id] else: local_si = deepcopy(local_best_si) local_f_array = f_array local_phases = phases local_si.b_blur_hires = test_b_blur_hires local_si.input_d_cut = test_d_cut local_si.b_iso = test_b_iso local_si.b_sharpen = original_b_iso-local_si.b_iso local_map_and_b = apply_sharpening( f_array = local_f_array, phases = local_phases, sharpening_info_obj = local_si, crystal_symmetry = local_si.crystal_symmetry, out = null_out()) local_si = score_map( map_data = local_map_and_b.map_data, sharpening_info_obj = local_si, out = null_out()) value_dict[id] = local_si.score if local_si.verbose: print(" %6.1f %6.1f %5s %7.3f %7.3f" %( local_si.b_sharpen, local_si.b_iso, local_si.b_blur_hires, local_si.adjusted_sa, local_si.kurtosis) + \ " %7.1f %7.3f %7.3f %7.3f " %( zero_if_none(local_si.adjusted_path_length), #local_si.sa_ratio, local_si.normalized_regions, test_d_cut, test_b_blur_hires ), file = out) if local_si.score > local_best_score: local_best_score = local_si.score local_best_working_si = deepcopy(local_si) if local_best_score > best_score: best_score = local_best_score working_best_si = deepcopy(local_best_working_si) delta_b_iso = delta_b_iso/2 delta_b_blur_hires = delta_b_blur_hires/2 delta_d_cut = delta_d_cut/2 print("Current working best "+\ "score = %7.3f b_iso = %5.1f b_blur_hires = %5.1f d_cut = %5.1f" %( working_best_si.score, working_best_si.b_iso, working_best_si.b_blur_hires, working_best_si.get_d_cut()), file = out) improving = True if working_best_si and working_best_si.score > local_best_si.score: print("Using new values of b_iso and b_blur_hires and d_cut", file = out) local_best_si = working_best_si local_best_si.show_summary(out = out) return local_best_si, local_best_map_and_b def set_mean_sd_of_map(map_data = None, target_mean = None, target_sd = None): if not map_data: return None new_mean = map_data.as_1d().min_max_mean().mean new_sd = max(1.e-10, map_data.sample_standard_deviation()) map_data = (map_data-new_mean)/new_sd # normalized return map_data*target_sd + target_mean # restore original def run_auto_sharpen( si = None, map_data = None, first_half_map_data = None, second_half_map_data = None, pdb_inp = None, auto_sharpen_methods = None, print_result = True, return_bsi = False, out = sys.stdout): if si.verbose: local_out = out else: local_out = null_out() # Identifies parameters for optimal map sharpening using analysis of density, # model-correlation, or half-map correlation (first_half_map_data vs # vs second_half_map_data). # NOTE: We can apply this to any map_data (a part or whole of the map) # BUT: need to update n_real if we change the part of the map! # change with map data: crystal_symmetry, solvent_fraction, n_real, wrapping, smoothed_box_mask_data = None original_box_map_data = None if si.auto_sharpen and ( si.box_in_auto_sharpen or si.density_select_in_auto_sharpen or pdb_inp): original_box_sharpening_info_obj = deepcopy(si) # should really not be box box_pdb_inp, box_map_data, box_first_half_map_data, \ box_second_half_map_data, \ box_crystal_symmetry, box_sharpening_info_obj, \ smoothed_box_mask_data, original_box_map_data, n_buffer = \ select_box_map_data(si = si, map_data = map_data, first_half_map_data = first_half_map_data, second_half_map_data = second_half_map_data, pdb_inp = pdb_inp, restrict_map_size = si.restrict_map_size, out = out, local_out = local_out) if box_sharpening_info_obj is None: # did not do it print("Box map is similar in size to entire map..."+\ "skipping representative box of density", file = out) original_box_sharpening_info_obj = None crystal_symmetry = si.crystal_symmetry else: print("Using small map to identify optimal sharpening", file = out) print("Box map grid: %d %d %d" %( box_map_data.all()), file = out) print("Box map cell: %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f "%( box_crystal_symmetry.unit_cell().parameters()), file = out) original_map_data = map_data original_crystal_symmetry = si.crystal_symmetry map_data = box_map_data pdb_inp = box_pdb_inp if si.density_select_in_auto_sharpen and ( # catch empty pdb_inp not pdb_inp or not \ pdb_inp.construct_hierarchy().overall_counts().n_residues): pdb_inp = None crystal_symmetry = box_crystal_symmetry if box_first_half_map_data: first_half_map_data = box_first_half_map_data if box_second_half_map_data: second_half_map_data = box_second_half_map_data # SET si for box now... si = deepcopy(si).update_with_box_sharpening_info( box_sharpening_info_obj = box_sharpening_info_obj) else: original_box_sharpening_info_obj = None box_sharpening_info_obj = None crystal_symmetry = si.crystal_symmetry starting_mean = map_data.as_1d().min_max_mean().mean starting_sd = map_data.sample_standard_deviation() print("\nGetting original b_iso...", file = out) map_coeffs_aa, map_coeffs, f_array, phases = effective_b_iso( map_data = map_data, resolution = si.resolution, d_min_ratio = si.d_min_ratio, scale_max = si.scale_max, remove_aniso = si.remove_aniso, crystal_symmetry = si.crystal_symmetry, out = out) original_b_iso = map_coeffs_aa.b_iso if original_b_iso is None: print("Could not determine original b_iso...setting to 200", file = out) original_b_iso = 200. si.original_aniso_obj = map_coeffs_aa # set it so we can apply it later if desired if first_half_map_data: first_half_map_coeffs, dummy = get_f_phases_from_map( map_data = first_half_map_data, crystal_symmetry = si.crystal_symmetry, d_min = si.resolution, d_min_ratio = si.d_min_ratio, remove_aniso = si.remove_aniso, scale_max = si.scale_max, return_as_map_coeffs = True, out = local_out) else: first_half_map_coeffs = None if second_half_map_data: second_half_map_coeffs, dummy = get_f_phases_from_map( map_data = second_half_map_data, crystal_symmetry = si.crystal_symmetry, d_min = si.resolution, d_min_ratio = si.d_min_ratio, scale_max = si.scale_max, remove_aniso = si.remove_aniso, return_as_map_coeffs = True, out = local_out) else: second_half_map_coeffs = None if pdb_inp: # Getting model information if pdb_inp present --------------------------- from cctbx.maptbx.refine_sharpening import get_model_map_coeffs_normalized model_map_coeffs = get_model_map_coeffs_normalized(pdb_inp = pdb_inp, si = si, f_array = f_array, resolution = si.resolution, out = out) if not model_map_coeffs: # give up pdb_inp = None if si.is_model_sharpening(): raise Sorry("Cannot carry out model sharpening without a model."+ " It could be that the model was outside the map") else: model_map_coeffs = None # Try various methods for sharpening. # XXX fix this up local_si = deepcopy(si).update_with_box_sharpening_info( box_sharpening_info_obj = box_sharpening_info_obj) if si.adjust_region_weight and \ (not si.sharpening_is_defined()) and (not si.is_model_sharpening()) \ and (not si.is_half_map_sharpening()) and ( not si.is_target_b_iso_to_d_cut()) and ( si.sharpening_target == 'adjusted_sa'): for iii in range(1): # just so we can break local_si = deepcopy(si).update_with_box_sharpening_info( box_sharpening_info_obj = box_sharpening_info_obj) local_si.sharpening_target = 'adjusted_sa' local_si.sharpening_method = 'b_iso_to_d_cut' sa_ratio_list = [] normalized_regions_list = [] if 0: #si.resolution: # 2017-07-26 reset b_low, b_mid, b_high, using 5.9*resolution**2 for b_mid delta_search = si.search_b_max-si.search_b_min b_mid = si.get_target_b_iso() b_low = b_mid-150*delta_search/400 b_high = b_mid+250*delta_search/400 print("Centering search on b_iso = %7.2f" %(b_mid), file = out) else: b_low = min(original_b_iso, si.search_b_min) b_high = max(original_b_iso, si.search_b_max) b_mid = b_low+0.375*(b_high-b_low) ok_region_weight = True results_list = [] kw_list = [] first = True id = 0 for b_iso in [b_low, b_high, b_mid]: id+= 1 if first and local_si.multiprocessing == 'multiprocessing' or \ local_si.nproc == 1: # can do anything local_log = out else: # skip log entirely local_log = None # will set this later and return as r.log_as_text first = False lsi = deepcopy(local_si) lsi.b_sharpen = original_b_iso-b_iso lsi.b_iso = b_iso # ------ SET UP RUN HERE ---------- kw_list.append( { 'f_array':f_array, 'phases':phases, 'crystal_symmetry':lsi.crystal_symmetry, 'local_si':lsi, 'id':id, 'out':local_log, }) # We are going to call autosharpening with this # ------ END OF SET UP FOR RUN ---------- """ local_map_and_b = apply_sharpening( f_array = f_array, phases = phases, sharpening_info_obj = lsi, crystal_symmetry = lsi.crystal_symmetry, out = null_out()) local_si = score_map(map_data = local_map_and_b.map_data, sharpening_info_obj = local_si, out = null_out()) """ # This is the actual run here ============= from libtbx.easy_mp import run_parallel results_list = run_parallel( method = si.multiprocessing, qsub_command = si.queue_run_command, nproc = si.nproc, target_function = run_sharpen_and_score, kw_list = kw_list) # results looks like: [result, result2] sort_list = [] for result in results_list: sort_list.append([result.id, result]) sort_list.sort(key=itemgetter(0)) for id, result in sort_list: local_si = result.local_si if local_si.sa_ratio is None or local_si.normalized_regions is None: ok_region_weight = False sa_ratio_list.append(local_si.sa_ratio) normalized_regions_list.append(local_si.normalized_regions) if not ok_region_weight: break # skip it # Set region weight so that either: # (1) delta_sa_ratio == region_weight*delta_normalized_regions # (2) sa_ratio = region_weight*normalized_regions (at low B) # region weight from change over entire region d_sa_ratio = sa_ratio_list[0]-sa_ratio_list[1] d_normalized_regions = normalized_regions_list[0]-normalized_regions_list[1] delta_region_weight = si.region_weight_factor*d_sa_ratio/max( 1.e-10, d_normalized_regions) if d_sa_ratio < 0 or d_normalized_regions < 0: print("Not using delta_region_weight with unusable values", file = out) ok_region_weight = False # region weight from initial values init_region_weight = si.region_weight_factor* \ sa_ratio_list[0]/max(1.e-10, normalized_regions_list[0]) # Ensure that adjusted_sa at b_mid is > than either end # adjusted_sa = sa_ratio - region_weight*normalized_regions # sa[2] = sa_ratio_list[2]-region_weight*normalized_regions[2] # sa[1] = sa_ratio_list[1]-region_weight*normalized_regions[1] # sa[0] = sa_ratio_list[0]-region_weight*normalized_regions[0] # sa[2] >= sa[1] and sa[2] >= sa[0] # sa_ratio_list[2]-region_weight*normalized_regions[2] >= # sa_ratio_list[1]-region_weight*normalized_regions[1] # NOTE: sa_ratio_list and normalized_regions both decrease in order: # low med high or [0] [2] [1] max_region_weight = (sa_ratio_list[2]- sa_ratio_list[1])/max(0.001, normalized_regions_list[2]-normalized_regions_list[1]) min_region_weight = (sa_ratio_list[0]- sa_ratio_list[2])/max(0.001, normalized_regions_list[0]-normalized_regions_list[2]) min_region_weight = max(1.e-10, min_region_weight) # positive max_region_weight = max(1.e-10, max_region_weight) # positive delta_weight = max(0., max_region_weight-min_region_weight) min_buffer = delta_weight*si.region_weight_buffer min_region_weight+= min_buffer max_region_weight-= min_buffer min_max_region_weight = True if min_region_weight >= max_region_weight: print("Warning: min_region_weight >= max_region_weight...", file = out) min_max_region_weight = False #ok_region_weight = False print("Region weight bounds: Min: %7.1f Max: %7.1f " %( min_region_weight, max_region_weight), file = out) print("Region weight estimates:", file = out) print("From ratio of low-B surface area to regions: %7.1f" %( init_region_weight), file = out) print("Ratio of change in surface area to change in regions: %7.1f" %( delta_region_weight), file = out) # put them in bounds but note if we did it out_of_range = False if ok_region_weight and si.region_weight_method == 'initial_ratio': if min_max_region_weight and ( init_region_weight > max_region_weight or \ init_region_weight max_region_weight or \ delta_region_weight1: b_min = min(original_b_iso, si.search_b_min) b_max = max(original_b_iso, si.search_b_max) else: # for just one, take it b_min = si.search_b_min b_max = si.search_b_max b_n = si.search_b_n delta_b = (b_max-b_min)/max(1, b_n-1) print("\nTesting %s with b_iso from %7.1f to %7.1f in %d steps of %7.1f" %( m, b_min, b_max, b_n, delta_b), file = out) print("(b_sharpen from %7.1f to %7.1f ) " %( original_b_iso-b_min, original_b_iso-b_max), file = out) if m == 'b_iso': k_sharpen = 0. else: k_sharpen = si.k_sharpen print("\nB-sharpen B-iso k_sharpen SA "+\ "Kurtosis Path len Normalized regions", file = out) # ------------------------ local_best_map_and_b = map_and_b_object() local_best_si = deepcopy(si).update_with_box_sharpening_info( box_sharpening_info_obj = box_sharpening_info_obj) si_b_iso_list = flex.double() si_score_list = flex.double() si_id_list = [] kw_list = [] first = True if return_bsi: assert local_si.nproc == 1 results_list = [] for i in range(b_n): #============================================ local_si = deepcopy(si).update_with_box_sharpening_info( box_sharpening_info_obj = box_sharpening_info_obj) local_si.sharpening_method = m local_si.n_real = map_data.all() local_si.k_sharpen = k_sharpen if first and local_si.multiprocessing == 'multiprocessing' or \ local_si.nproc == 1: # can do anything local_log = out else: # skip log entirely local_log = None # will set this later and return as r.log_as_text first = False if m == 'resolution_dependent': print("\nRefining resolution-dependent sharpening based on %s" %( local_si.residual_target), file = out) local_si.b_sharpen = 0 local_si.b_iso = original_b_iso from cctbx.maptbx.refine_sharpening import run as refine_sharpening local_f_array, local_phases = refine_sharpening( map_coeffs = map_coeffs, sharpening_info_obj = local_si, out = out) elif m == 'model_sharpening': print("\nUsing model-based sharpening", file = out) local_si.b_sharpen = 0 local_si.b_iso = original_b_iso from cctbx.maptbx.refine_sharpening import scale_amplitudes scale_amplitudes( model_map_coeffs = model_map_coeffs, map_coeffs = map_coeffs, si = local_si, out = out) # local_si contains target_scale_factors now local_f_array = f_array local_phases = phases elif m == 'half_map_sharpening': print("\nUsing half-map-based sharpening", file = out) local_si.b_sharpen = 0 local_si.b_iso = original_b_iso from cctbx.maptbx.refine_sharpening import scale_amplitudes scale_amplitudes( model_map_coeffs = model_map_coeffs, map_coeffs = map_coeffs, first_half_map_coeffs = first_half_map_coeffs, second_half_map_coeffs = second_half_map_coeffs, si = local_si, out = out) # local_si contains target_scale_factors now local_f_array = f_array local_phases = phases elif m == 'external_map_sharpening': print("\nUsing external-map-based sharpening", file = out) local_si.b_sharpen = 0 local_si.b_iso = original_b_iso from cctbx.maptbx.refine_sharpening import scale_amplitudes scale_amplitudes( model_map_coeffs = model_map_coeffs, map_coeffs = map_coeffs, external_map_coeffs = first_half_map_coeffs, si = local_si, out = out) # local_si contains target_scale_factors now local_f_array = f_array local_phases = phases else: local_f_array = f_array local_phases = phases b_iso = b_min+i*delta_b local_si.b_sharpen = original_b_iso-b_iso local_si.b_iso = b_iso # ------ SET UP RUN HERE ---------- kw_list.append( { 'f_array':local_f_array, 'phases':local_phases, 'crystal_symmetry':local_si.crystal_symmetry, 'original_b_iso':original_b_iso, 'local_si':local_si, 'm':m, 'return_bsi':return_bsi, 'out':local_log, 'id':i+1, }) # We are going to call autosharpening with this # ------ END OF SET UP FOR RUN ---------- # This is the actual run here ============= from libtbx.easy_mp import run_parallel results_list = run_parallel( method = si.multiprocessing, qsub_command = si.queue_run_command, nproc = si.nproc, target_function = run_sharpen_and_score, kw_list = kw_list) # results looks like: [result, result2] sort_list = [] for result in results_list: sort_list.append([result.id, result]) sort_list.sort(key=itemgetter(0)) for id, result in sort_list: local_si = result.local_si local_map_and_b = result.local_map_and_b if result.text: print(result.text, file = out) # Run through all result to get these if local_si.b_sharpen is not None and local_si.b_iso is not None and\ local_si.k_sharpen is not None and local_si.kurtosis is not None \ and local_si.adjusted_sa is not None and local_si.score is not None: si_b_iso_list.append(local_si.b_iso) si_score_list.append(local_si.score) if local_si.k_sharpen is not None: si_id_list.append("%.3f_%.3f_%.3f" %( local_si.b_iso, local_si.k_sharpen, local_si.get_d_cut())) if m == 'no_sharpening': null_si = local_si if local_best_si.score is None or local_si.score>local_best_si.score: local_best_si = local_si local_best_map_and_b = local_map_and_b # ============================================ # DONE WITH ALL RUNS if not local_best_si.is_model_sharpening() and \ not local_best_si.is_half_map_sharpening(): if local_best_si.sharpening_method == 'resolution_dependent': print("\nBest scores for sharpening with "+\ "b[0] = %6.2f b[1] = %6.2f b[2] = %6.2f: " %( local_best_si.resolution_dependent_b[0], local_best_si.resolution_dependent_b[1], local_best_si.resolution_dependent_b[2]), file = out) else: print("\nBest scores for sharpening with "+\ "b_iso = %6.1f b_sharpen = %6.1f k_sharpen = %s: " %( local_best_si.b_iso, local_best_si.b_sharpen, local_best_si.k_sharpen), file = out) if local_best_si.score is not None: local_best_si.show_summary(out = out) print("Adjusted surface area: %7.3f Kurtosis: %7.3f Score: %7.3f\n" %( local_best_si.adjusted_sa, local_best_si.kurtosis, local_best_si.score), file = out) if si_score_list.size()>1: # test for signal signal_to_noise = estimate_signal_to_noise(value_list = si_score_list) print("Estimated signal-to-noise in ID of optimal sharpening: %5.1f" %( signal_to_noise), file = out) if signal_to_noise best_si.score): best_si = local_best_si best_map_and_b = local_best_map_and_b if not best_si.is_model_sharpening() and \ not best_si.is_half_map_sharpening(): print("This is the current best score\n", file = out) if (best_si.score is not None ) and ( not best_si.is_model_sharpening() ) and (not best_si.is_half_map_sharpening()): print("\nOverall best sharpening method: %s Score: %7.3f\n" %( best_si.sharpening_method, best_si.score), file = out) best_si.show_summary(out = out) if (not best_si.is_model_sharpening()) and \ (not best_si.is_half_map_sharpening()) and null_si: if best_si.score>null_si.score: # we improved them.. print("Improved score with sharpening...", file = out) else: print("Did not improve score with sharpening...", file = out) if return_bsi: map_data = best_map_and_b.map_data map_data = set_mean_sd_of_map(map_data = map_data, target_mean = starting_mean, target_sd = starting_sd) box_sharpening_info_obj.map_data = map_data box_size= map_data.all() calculated_box_size=tuple([i-j+1 for i,j in zip( box_sharpening_info_obj.upper_bounds, box_sharpening_info_obj.lower_bounds)]) calculated_box_size_minus_one=tuple([i-j for i,j in zip( box_sharpening_info_obj.upper_bounds, box_sharpening_info_obj.lower_bounds)]) if calculated_box_size_minus_one == box_size: # one too big box_sharpening_info_obj.upper_bounds =tuple( [i-1 for i in box_sharpening_info_obj.upper_bounds] ) # work-around for upper bounds off by one in model sharpening box_sharpening_info_obj.smoothed_box_mask_data = smoothed_box_mask_data box_sharpening_info_obj.original_box_map_data = original_box_map_data box_sharpening_info_obj.n_buffer = n_buffer box_sharpening_info_obj.crystal_symmetry = best_si.crystal_symmetry box_sharpening_info_obj.resolution = best_si.resolution box_sharpening_info_obj.d_min_ratio = best_si.d_min_ratio box_sharpening_info_obj.scale_max = best_si.scale_max box_sharpening_info_obj.smoothing_radius = best_si.smoothing_radius box_sharpening_info_obj.b_iso = best_map_and_b.final_b_iso box_sharpening_info_obj.starting_b_iso = best_map_and_b.starting_b_iso return box_sharpening_info_obj if original_box_sharpening_info_obj: # Put back original crystal_symmetry with original_box_sharpening_info_obj print("\nRestoring original symmetry to best sharpening info", file = out) best_si.update_with_box_sharpening_info( box_sharpening_info_obj = original_box_sharpening_info_obj) print("(%7.3f, %7.3f, %7.3f, %7.3f, %7.3f, %7.3f) "%(tuple( best_si.crystal_symmetry.unit_cell().parameters())), file = out) # and set tracking data with result return best_si def run_sharpen_and_score(f_array = None, phases = None, local_si = None, crystal_symmetry = None, original_b_iso = None, m = None, return_bsi = None, id = None, out = sys.stdout): local_map_and_b = apply_sharpening( f_array = f_array, phases = phases, sharpening_info_obj = local_si, crystal_symmetry = crystal_symmetry, out = null_out()) local_si = score_map(map_data = local_map_and_b.map_data, sharpening_info_obj = local_si, out = null_out()) # Record b_iso values if not local_map_and_b.starting_b_iso: local_map_and_b.starting_b_iso = original_b_iso if not local_map_and_b.final_b_iso: local_map_and_b.final_b_iso = local_si.b_iso # This is printout below here =============== if m == 'resolution_dependent': text = \ "\nb[0] b[1] b[2] SA Kurtosis sa_ratio Normalized regions" text+= "\n"+\ "\nB-sharpen B-iso k_sharpen SA "+\ "Kurtosis Path len Normalized regions" text+= "\n"+" %6.2f %6.2f %6.2f " %( local_si.resolution_dependent_b[0], local_si.resolution_dependent_b[1], local_si.resolution_dependent_b[2]) +\ " %7.3f %7.3f " %( local_si.adjusted_sa, local_si.kurtosis)+\ " %7.1f %7.3f" %( zero_if_none(local_si.adjusted_path_length), #local_si.sa_ratio, local_si.normalized_regions) elif local_si.b_sharpen is not None and local_si.b_iso is not None and\ local_si.k_sharpen is not None and local_si.kurtosis is not None \ and local_si.adjusted_sa is not None: text = \ " %6.1f %6.1f %5s %7.3f %7.3f" %( local_si.b_sharpen, local_si.b_iso, local_si.k_sharpen, local_si.adjusted_sa, local_si.kurtosis) + \ " %7.1f %7.3f" %( zero_if_none(local_si.adjusted_path_length), #local_si.sa_ratio, local_si.normalized_regions) else: text = "" if return_bsi: r = group_args( local_si = local_si, local_map_and_b = local_map_and_b, text = text, id = id) else: r = group_args( local_si = local_si, local_map_and_b = None, text = text, id = id) return r def effective_b_iso(map_data = None, tracking_data = None, box_sharpening_info_obj = None, crystal_symmetry = None, resolution = None, remove_aniso = None, d_min_ratio = None, scale_max = None, out = sys.stdout): if not crystal_symmetry: if box_sharpening_info_obj: crystal_symmetry = box_sharpening_info_obj.crystal_symmetry else: crystal_symmetry = tracking_data.crystal_symmetry if resolution: d_min = resolution else: d_min = tracking_data.params.crystal_info.resolution if not d_min_ratio: d_min_ratio = tracking_data.params.map_modification.d_min_ratio map_coeffs, map_coeffs_ra = get_f_phases_from_map(map_data = map_data, crystal_symmetry = crystal_symmetry, d_min = d_min, d_min_ratio = d_min_ratio, scale_max = scale_max, remove_aniso = remove_aniso, return_as_map_coeffs = True, out = out) f_array, phases = map_coeffs_as_fp_phi(map_coeffs) if map_coeffs_ra: b_iso = map_coeffs_ra.b_iso else: b_iso = None if b_iso is not None: print("Effective B-iso = %7.2f\n" %(b_iso), file = out) else: print("Effective B-iso not determined\n", file = out) return map_coeffs_ra, map_coeffs, f_array, phases def update_tracking_data_with_sharpening(map_data = None, tracking_data = None, si = None, out = sys.stdout): # Set shifted_map_info if map_data is new if tracking_data.params.output_files.shifted_sharpened_map_file: shifted_sharpened_map_file = os.path.join( tracking_data.params.output_files.output_directory, tracking_data.params.output_files.shifted_sharpened_map_file) else: shifted_sharpened_map_file = None from cctbx.maptbx.segment_and_split_map import write_ccp4_map if shifted_sharpened_map_file: write_ccp4_map(tracking_data.crystal_symmetry, shifted_sharpened_map_file, map_data) print("Wrote shifted, sharpened map to %s" %( shifted_sharpened_map_file), file = out) tracking_data.set_shifted_map_info(file_name = shifted_sharpened_map_file, crystal_symmetry = tracking_data.crystal_symmetry, origin = map_data.origin(), all = map_data.all(), b_sharpen = None) def get_high_points_from_map( map_data = None, boundary_radius = 5., unit_cell = None, out = sys.stdout): max_in_map_data = map_data.as_1d().min_max_mean().max for cutoff in [0.99, 0.98, 0.95, 0.90, 0.50]: high_points_mask = (map_data>= cutoff*max_in_map_data) sda = map_data.as_1d().min_max_mean().max for nth_point in [4, 2, 1]: sites_cart = get_marked_points_cart(mask_data = high_points_mask, unit_cell = unit_cell, every_nth_point = nth_point, boundary_radius = boundary_radius) if sites_cart.size()>0: break if sites_cart.size()>0: break assert sites_cart.size()>0 del high_points_mask sites_cart = sites_cart[:1] xyz_frac = unit_cell.fractionalize(sites_cart[0]) value = map_data.value_at_closest_grid_point(xyz_frac) print("High point in map at (%7.2f %7.2f %7.2f) with value of %7.2f " %( sites_cart[0][0], sites_cart[0][1], sites_cart[0][2], value), file = out) return sites_cart def get_one_au(tracking_data = None, sites_cart = None, ncs_obj = None, map_data = None, starting_mask = None, radius = None, every_nth_point = None, removed_ncs = None, out = sys.stdout): unit_cell = tracking_data.crystal_symmetry.unit_cell() if removed_ncs: # take everything left mm = map_data.as_1d().min_max_mean() mask_threshold = mm.min+max(0.00001, 0.0001*(mm.mean-mm.min)) # just above min else: mask_threshold = tracking_data.params.segmentation.mask_threshold every_nth_point = tracking_data.params.segmentation.grid_spacing_for_au radius = tracking_data.params.segmentation.radius if not radius: radius = set_radius(unit_cell = unit_cell, map_data = map_data, every_nth_point = every_nth_point) tracking_data.params.segmentation.radius = radius print("\nRadius for AU identification: %7.2f A" %(radius), file = out) overall_mask, max_in_sd_map, sd_map = get_overall_mask(map_data = map_data, mask_threshold = mask_threshold, crystal_symmetry = tracking_data.crystal_symmetry, resolution = tracking_data.params.crystal_info.resolution, solvent_fraction = tracking_data.solvent_fraction, radius = radius, out = out) if starting_mask: print("Points in starting mask:", starting_mask.count(True), file = out) print("Points in overall mask:", overall_mask.count(True), file = out) print("Points in both:", (starting_mask & overall_mask).count(True), file = out) if tracking_data.params.crystal_info.is_crystal: # take starting mask as overall... overall_mask = starting_mask else: # usual # make sure overall mask is at least as big.. overall_mask = (overall_mask | starting_mask) print("New size of overall mask: ", overall_mask.count(True), file = out) else: if not sites_cart: # pick top of map sites_cart = get_high_points_from_map( boundary_radius = radius, map_data = sd_map, unit_cell = unit_cell, out = out) starting_mask = mask_from_sites_and_map( # starting au mask map_data = sd_map, unit_cell = unit_cell, sites_cart = sites_cart, radius = radius, overall_mask = overall_mask) del sd_map au_mask, ncs_mask = get_ncs_mask( map_data = map_data, unit_cell = unit_cell, ncs_object = ncs_obj, starting_mask = starting_mask, radius = radius, overall_mask = overall_mask, every_nth_point = every_nth_point) print("Points in au: %d in ncs: %d (total %7.1f%%) both: %d Not marked: %d" %( au_mask.count(True), ncs_mask.count(True), 100.*float(au_mask.count(True)+ncs_mask.count(True))/au_mask.size(), (au_mask & ncs_mask).count(True), au_mask.size()-au_mask.count(True)-ncs_mask.count(True), ), file = out) return au_mask def set_up_sharpening(si = None, map_data = None, out = sys.stdout): print("\nCarrying out specified sharpening/blurring of map", file = out) check_si = si # just use input information check_si.show_summary(out = out) if check_si.is_target_b_iso_to_d_cut(): check_si.b_iso = check_si.get_target_b_iso() check_si.b_sharpen = None print("Setting target b_iso of %7.1f " %(check_si.b_iso), file = out) if check_si.b_sharpen is None and check_si.b_iso is not None: # need to figure out b_sharpen print("\nGetting b_iso of map", file = out) b_iso = check_si.get_effective_b_iso(map_data = map_data, out = out) check_si.b_sharpen = b_iso-check_si.b_iso # sharpen is what to print("Value of b_sharpen to obtain b_iso of %s is %5.2f" %( check_si.b_iso, check_si.b_sharpen), file = out) elif check_si.b_sharpen is not None: print("Sharpening b_sharpen will be %s" %(check_si.b_sharpen), file = out) elif check_si.resolution_dependent_b: print("Resolution-dependent b_sharpening values:" +\ "b0: %7.2f b1: %7.2f b2: %7.2f " %( tuple(check_si.resolution_dependent_b)), file = out) elif check_si.target_scale_factors: print("Model sharpening scale values:", file = out) for x in check_si.target_scale_factors: print(x, end = ' ', file = out) print(file = out) return check_si def run(args, params = None, map_data = None, crystal_symmetry = None, write_files = None, auto_sharpen = None, density_select = None, add_neighbors = None, save_box_map_ncs_au = None, resolution = None, sequence = None, half_map_data_list = None, ncs_obj = None, tracking_data = None, target_scattered_points = None, is_iteration = False, pdb_hierarchy = None, target_xyz = None, target_hierarchy = None, target_model = None, sharpening_target_pdb_inp = None, wrapping = None, target_ncs_au_file = None, regions_to_keep = None, solvent_content = None, molecular_mass = None, symmetry = None, chain_type = None, keep_low_density = None, box_buffer = None, soft_mask_extract_unique = None, mask_expand_ratio = None, out = sys.stdout): if is_iteration: print("\nIteration tracking data:", file = out) tracking_data.show_summary(out = out) else: # get the parameters and map_data (sharpened, magnified, shifted...) params, map_data, half_map_data_list, pdb_hierarchy, tracking_data, \ shifted_ncs_object = get_params( # args, map_data = map_data, crystal_symmetry = crystal_symmetry, half_map_data_list = half_map_data_list, ncs_object = ncs_obj, write_files = write_files, auto_sharpen = auto_sharpen, density_select = density_select, add_neighbors = add_neighbors, save_box_map_ncs_au = save_box_map_ncs_au, sequence = sequence, wrapping = wrapping, target_ncs_au_file = target_ncs_au_file, regions_to_keep = regions_to_keep, solvent_content = solvent_content, resolution = resolution, molecular_mass = molecular_mass, symmetry = symmetry, chain_type = chain_type, keep_low_density = keep_low_density, box_buffer = box_buffer, soft_mask_extract_unique = soft_mask_extract_unique, mask_expand_ratio = mask_expand_ratio, sharpening_target_pdb_inp = sharpening_target_pdb_inp, out = out) if params.control.shift_only: return map_data, ncs_obj, tracking_data elif params.control.check_ncs or \ params.control.sharpen_only: return None, None, tracking_data if params.input_files.pdb_to_restore: restore_pdb(params, tracking_data = tracking_data, out = out) return None, None, tracking_data # read and write the ncs (Normally point-group NCS) ncs_obj, tracking_data = get_ncs(params = params, tracking_data = tracking_data, ncs_object = shifted_ncs_object, out = out) if target_model: target_hierarchy = target_model.get_hierarchy() elif params.input_files.target_ncs_au_file: # read in target import iotbx.pdb target_hierarchy = iotbx.pdb.input( file_name = params.input_files.target_ncs_au_file).construct_hierarchy() print("\nShifting model based on origin shift (if any)", file = out) print("Coordinate shift is (%7.2f, %7.2f, %7.2f)" %( tuple(tracking_data.origin_shift)), file = out) if not map_data: raise Sorry("Need map data for segment_and_split_map") if params.output_files.shifted_map_file: shifted_map_file = os.path.join( tracking_data.params.output_files.output_directory, params.output_files.shifted_map_file) else: shifted_map_file = None if params.output_files.shifted_ncs_file: shifted_ncs_file = os.path.join( tracking_data.params.output_files.output_directory, params.output_files.shifted_ncs_file) else: shifted_ncs_file = None if params.output_files.shifted_ncs_file: shifted_pdb_file = os.path.join( tracking_data.params.output_files.output_directory, params.output_files.shifted_pdb_file) else: shifted_pdb_file = None ncs_obj, pdb_hierarchy, target_hierarchy, \ tracking_data, sharpening_target_pdb_inp = apply_origin_shift( shifted_map_file = shifted_map_file, shifted_pdb_file = shifted_pdb_file, shifted_ncs_file = shifted_ncs_file, origin_shift = tracking_data.origin_shift, shifted_ncs_object = shifted_ncs_object, pdb_hierarchy = pdb_hierarchy, target_hierarchy = target_hierarchy, map_data = map_data, tracking_data = tracking_data, sharpening_target_pdb_inp = sharpening_target_pdb_inp, out = out) if target_hierarchy: target_xyz = target_hierarchy.atoms().extract_xyz() del target_hierarchy # We can use params.input_files.target_ncs_au_file here to define ncs au if target_xyz and not target_scattered_points: target_scattered_points = flex.vec3_double() target_scattered_points.append(target_xyz.mean()) # get the chain types and therefore (using ncs_copies) volume fraction tracking_data = get_solvent_fraction(params, ncs_object = ncs_obj, tracking_data = tracking_data, out = out) # Done with getting params and maps # Summarize after any sharpening tracking_data.show_summary(out = out) original_ncs_obj = ncs_obj # in case we need it later... original_input_ncs_info = tracking_data.input_ncs_info removed_ncs = False n_residues = tracking_data.n_residues ncs_copies = tracking_data.input_ncs_info.number_of_operators if (not tracking_data.solvent_fraction) and \ params.crystal_info.molecular_mass: tracking_data.solvent_fraction = get_solvent_fraction_from_molecular_mass( crystal_symmetry = tracking_data.crystal_symmetry, molecular_mass = params.crystal_info.molecular_mass, out = out) if tracking_data.solvent_fraction: solvent_fraction = tracking_data.solvent_fraction else: raise Sorry("Need solvent fraction or molecular mass or sequence file") # Now usual method, using our new map...should duplicate best result above for itry in range(2): # get connectivity (conn = connectivity_object.result) b_vs_region = b_vs_region_info() si = sharpening_info(tracking_data = tracking_data) co, sorted_by_volume, min_b, max_b, unique_expected_regions, best_score, \ new_threshold, starting_density_threshold = \ get_connectivity( b_vs_region = b_vs_region, map_data = map_data, iterate_with_remainder = params.segmentation.iterate_with_remainder, n_residues = n_residues, ncs_copies = ncs_copies, solvent_fraction = solvent_fraction, fraction_occupied = si.fraction_occupied, min_volume = si.min_volume, min_ratio = si.min_ratio, wrapping = si.wrapping, residues_per_region = si.residues_per_region, max_ratio_to_target = si.max_ratio_to_target, min_ratio_to_target = si.min_ratio_to_target, min_ratio_of_ncs_copy_to_first = si.min_ratio_of_ncs_copy_to_first, starting_density_threshold = si.starting_density_threshold, density_threshold = si.density_threshold, crystal_symmetry = si.crystal_symmetry, chain_type = si.chain_type, verbose = si.verbose, out = out) params.segmentation.starting_density_threshold = starting_density_threshold # have to set tracking data as we are passing that above tracking_data.params.segmentation.starting_density_threshold = starting_density_threshold # have to set tracking data as we are passing that above if new_threshold: print("\nNew threshold is %7.2f" %(new_threshold), file = out) if co is None: # no luck return None, None, tracking_data # Check to see which regions are in more than one au of the NCS # and set them aside. Group ncs-related regions together ncs_group_obj, tracking_data, equiv_dict_ncs_copy = identify_ncs_regions( params, sorted_by_volume = sorted_by_volume, co = co, min_b = min_b, max_b = max_b, ncs_obj = ncs_obj, tracking_data = tracking_data, out = out) if ncs_group_obj and ncs_group_obj.ncs_group_list: # ok break elif ncs_obj and itry == 0 and not is_iteration:# try again print("No NCS groups identified on first try...taking entire NCS AU.", file = out) # Identify ncs au au_mask = get_one_au(tracking_data = tracking_data, ncs_obj = ncs_obj, map_data = map_data, out = out) s = (au_mask == False) min_in_map = map_data.as_1d().min_max_mean().min map_data.set_selected(s, min_in_map) # mask out all but au from mmtbx.ncs.ncs import ncs ncs_obj = ncs() ncs_obj.set_unit_ncs() tracking_data.set_ncs_obj(ncs_obj = None) tracking_data.update_ncs_info(number_of_operators = 1) if n_residues: n_residues = n_residues/ncs_copies solvent_fraction = max(0.001, min(0.999, 1-((1-solvent_fraction)/ncs_copies))) ncs_copies = 1 params.segmentation.require_complete = False params.segmentation.iterate_with_remainder = False # so we do not iterate removed_ncs = True # Run again else: # tried twice, give up return None, None, tracking_data # Choose one region or group of regions from each ncs_group in the list # Optimize the closeness of centers # Select group of regions that are close together and represent one au ncs_group_obj, scattered_points = \ select_regions_in_au( params, ncs_group_obj = ncs_group_obj, equiv_dict_ncs_copy = equiv_dict_ncs_copy, tracking_data = tracking_data, target_scattered_points = target_scattered_points, unique_expected_regions = unique_expected_regions, out = out) # write out mask and map for all the selected regions... # Iterate if desired if params.segmentation.iterate_with_remainder and \ ncs_group_obj.selected_regions: print("\nCreating remaining mask and map", file = out) map_data_remaining = create_remaining_mask_and_map(params, ncs_group_obj = ncs_group_obj, map_data = map_data, crystal_symmetry = tracking_data.crystal_symmetry, out = out) remainder_ncs_group_obj = iterate_search(params, map_data = map_data, map_data_remaining = map_data_remaining, ncs_obj = ncs_obj, ncs_group_obj = ncs_group_obj, scattered_points = scattered_points, tracking_data = tracking_data, out = out) else: remainder_ncs_group_obj = None # collect all NCS ops that are needed to relate all the regions # that are used ncs_ops_used = ncs_group_obj.ncs_ops_used if remainder_ncs_group_obj and remainder_ncs_group_obj.ncs_ops_used: for x in remainder_ncs_group_obj.ncs_ops_used: if not x in ncs_ops_used: ncs_ops_used.append(x) if ncs_ops_used: ncs_ops_used.sort() print("Final NCS ops used: ", ncs_ops_used, file = out) # Save the used NCS ops ncs_used_obj = ncs_group_obj.ncs_obj.deep_copy(ops_to_keep = ncs_ops_used) if params.output_files.shifted_used_ncs_file: shifted_used_ncs_file = os.path.join( tracking_data.params.output_files.output_directory, params.output_files.shifted_used_ncs_file) ncs_used_obj.format_all_for_group_specification( file_name = shifted_used_ncs_file) tracking_data.set_shifted_used_ncs_info(file_name = shifted_used_ncs_file, number_of_operators = ncs_used_obj.max_operators(), is_helical_symmetry = tracking_data.input_ncs_info.is_helical_symmetry) tracking_data.shifted_used_ncs_info.show_summary(out = out) # Write out final maps and dummy atom files if params.output_files.write_output_maps: print("\nWriting output maps", file = out) else: print("\nSetting up but not writing output maps", file = out) map_files_written = write_output_files(params, tracking_data = tracking_data, map_data = map_data, half_map_data_list = half_map_data_list, ncs_group_obj = ncs_group_obj, remainder_ncs_group_obj = remainder_ncs_group_obj, pdb_hierarchy = pdb_hierarchy, removed_ncs = removed_ncs, out = out) ncs_group_obj.set_map_files_written(map_files_written) # Restore ncs info if we removed it if removed_ncs: print("\nRestoring original NCS info to tracking_data", file = out) tracking_data.input_ncs_info = original_input_ncs_info if params.output_files.output_info_file and ncs_group_obj: write_info_file(params = params, tracking_data = tracking_data, out = out) return ncs_group_obj, remainder_ncs_group_obj, tracking_data if __name__ == "__main__": run(args = sys.argv[1:])