We launched new forums in March 2019—join us there. In a hurry for help with your website? Get Help Now!
    • 40735
    • 119 Posts
    Hi!
    We're getting our feet wet with xPDO on a small project where we're retrieving a list of names of graduates. There are about 4500 records in our table and we want to "select distinct city from ttu_dglist_names" but it take upwards of 30 seconds to load the page and does not select anything distinct. Sometimes it gives us out of memory errors. What are we doing wrong?

    This is the specific bit of code giving us trouble. At the bottom is the whole script in case it helps.
    $query = $modx->newQuery("Names");
    	$query->distinct();
    	$query->select(array($modx->getSelectColumns("Names", array("city"))));
    	$query->where(array("category"=>$list_full));
    	$query->andCondition(array("termcode"=>$term));
    	$query->sortby("city", "ASC");
    	
    	$cities = $modx->getCollection("Names", $query);
    	foreach($cities as $city)
    		$cityOptions .= $modx->getChunk("deangradlist-menu-city", array("city"=>$city->get("city")));


    Schema:
    <?xml version="1.0" encoding="UTF-8"?>
    <model package="ttu_dglist" baseClass="xPDOObject" platform="mysql" defaultEngine="MyISAM" version="1.1">
    	<object class="Names" table="names" extends="xPDOSimpleObject">
    		<field key="firstname" dbtype="varchar" precision="250" phptype="string" null="false" />
    		<field key="middlename" dbtype="varchar" precision="250" phptype="string" null="false" />
    		<field key="lastname" dbtype="varchar" precision="250" phptype="string" null="false" />
    		<field key="city" dbtype="varchar" precision="250" phptype="string" null="false" />
    		<field key="county" dbtype="varchar" precision="250" phptype="string" null="false" />
    		<field key="state" dbtype="varchar" precision="250" phptype="string" null="false" />
    		<field key="termcode" dbtype="varchar" precision="250" phptype="string" null="false" />
    		<field key="category" dbtype="varchar" precision="250" phptype="string" null="false" />
    	</object>
    	<object class="Terms" table="terms" extends="xPDOObject">
    		<field key="termcode" dbtype="varchar" precision="6" phptype="string" null="false" index="pk" />
    		<field key="description" dbtype="varchar" precision="20" phptype="string" null="false" />
    		<field key="published" dbtype="tinyint" precision="1" phptype="integer" null="false" default="0" />
    
    		<index alias="PRIMARY" name="PRIMARY" primary="true" unique="true" type="BTREE" >
    			<column key="termcode" length="" collation="A" null="false" />
    		</index>
    	</object>
    </model>


    CREATE TABLE:
    CREATE TABLE `ttu_dglist_names` (
     `id` int(11) NOT NULL AUTO_INCREMENT,
     `firstname` varchar(250) NOT NULL,
     `middlename` varchar(250) NOT NULL,
     `lastname` varchar(250) NOT NULL,
     `city` varchar(250) NOT NULL,
     `county` varchar(250) NOT NULL,
     `state` varchar(250) NOT NULL,
     `termcode` varchar(250) NOT NULL,
     `category` varchar(250) NOT NULL,
     PRIMARY KEY (`id`)
    ) ENGINE=MyISAM AUTO_INCREMENT=4646 DEFAULT CHARSET=latin1


    Full Snippet:
    <?php $modx->regClientCSS(MODX_ASSETS_URL."templates/ttuv3/css/ttu_dglist.css");
    
    $output = "";
    $items = "";
    $query = "";
    $list_full = "";
    $hdr = "";
    $row = 1;
    $termslist = "";
    $cityOptions = "";
    $regionOptions = "";
    $path = MODX_CORE_PATH . "components/ttu_dglist/model/";
    
    if(!$modx->addPackage('ttu_dglist', $path, 'ttu_dglist_'))
        return 'There was a problem adding your package!  Check the logs for more info!';
    
    if(!isset($_GET["term"]) || !isset($_GET["list"])){
    	$query = $modx->newQuery("Terms");
    	$query->where(array("published", 1));
    	$query->sortby("termcode", "DESC");
    	$terms = $modx->getCollection("Terms", $query);
    	
    	foreach($terms as $term)
    		$termslist .= $modx->getChunk("deangradterm", array("description"=>$term->get("description"), "termcode"=>$term->get("termcode")));
    	
    	$output = $modx->getChunk("deangradterms", array("termslist"=>$termslist));
    }else{
    	$term = $_GET["term"];
    	$list = $_GET["list"];
    
    	//get term name...
    	$query = $modx->newQuery('Terms');
    	$query->where(array('termcode:=' => $term) ); 
    	$result = $modx->getObject('Terms', $query);
    	$tname = $result->get('description');
    	
    	//set full category name and header variable...
    	if ($list == "dean") { $list_full = "deanslist";
    						  $hdr = $tname . " Dean's List";}
    	elseif ($list == "grad") { $list_full = "gradlist"; 
    							  $hdr = $tname . " Graduate's List"; }
    	
    	//query the Names table...
    	$query = $modx->newQuery('Names');
    	$query->where(array('category:=' => $list_full));
    	$query->andCondition(array('termcode' => $term));
    	$query->sortby('lastname','ASC');
    	$query->sortby('firstname','ASC');
    	$query->sortby('middlename','ASC');
    	 
    	$total = $modx->getCount('Names',$query);
    	$totalVar = $modx->getOption('totalVar', $scriptProperties, 'total');
    	$modx->setPlaceholder($totalVar,$total);
    	
    	//set the limit and offset; use in getCollection command...
    	$limit = 25;     //$limit = $modx->getOption('limit',$scriptProperties,100);
    	$offset = $modx->getOption('offset',$scriptProperties,0);
    	$query->limit($limit,$offset);
    	
    	$result = $modx->getCollection('Names', $query);
    	$output = '';
    	
    	if($result)
    	{
    		  foreach($result as $row)
    		  {
    				$items .= $modx->getChunk("deangraditem", array("firstname"=>$row->get('firstname'), "middlename"=>$row->get('middlename'), "lastname"=>$row->get('lastname'),
    										  "city"=>$row->get('city'), "county"=>$row->get('county'), "state"=>$row->get('state'), "which"=>$row->get('category')));
    		  }
    	}
    	else {
    		return error_get_last();
    	}
    	
    	$query = $modx->newQuery("Names");
    	$query->distinct();
    	$query->select(array($modx->getSelectColumns("Names", array("city"))));
    	$query->where(array("category"=>$list_full));
    	$query->andCondition(array("termcode"=>$term));
    	$query->sortby("city", "ASC");
    	
    	$cities = $modx->getCollection("Names", $query);
    	foreach($cities as $city)
    		$cityOptions .= $modx->getChunk("deangradlist-menu-city", array("city"=>$city->get("city")));
    	
    	$output = $modx->getChunk("deangradlist", array("items"=>$items, "hdr"=>$hdr, "regionOptions"=>$regionOptions, "cityOptions"=>$cityOptions));
    }
    
    return $output;
    ?>
      • 40735
      • 119 Posts
      BTW, here are the specific errors that I get:

      Warning: trim() expects parameter 1 to be string, array given in /var/www/html/core/xpdo/xpdo.class.php on line 2075

      Fatal error: Allowed memory size of 134217728 bytes exhausted (tried to allocate 266149 bytes) in /var/www/html/core/model/modx/modparser.class.php on line 120
        • 22303 MODX Staff
        • 10,725 Posts
        You are working with too many records in memory at a time. How many rows that meet the query criteria are being returned? You could look at using getIterator() instead of getCollection() to avoid loading every row into memory at once, though you are still aggregating the the getChunk() output for each row into a variable...
          • 40735
          • 119 Posts
          Quote from: opengeek at Feb 12, 2013, 01:38 PM
          You are working with too many records in memory at a time. How many rows that meet the query criteria are being returned? You could look at using getIterator() instead of getCollection() to avoid loading every row into memory at once, though you are still aggregating the the getChunk() output for each row into a variable...

          When I run the query from MySQL Workbench it returns 468 rows however we're returning a lot more than that 3988 when we output the actual names and that seems to work fine, although it loads slowly. Are we taking the wrong approach to this? I have the impression from what you said that aggregating getChunk() output isn't ideal. We're trying to populate a <select> element with <option> elements for each city in the database. Is there another way to do that? Should we put the HTML in our snippet and not use a chunk? Also, if it makes a difference, we're using getPage for pagination. I'm not sure how that's affecting things. We're calling it like so:

          [[!getPage@ttu_getpage_properties? &elementClass=`modSnippet` &element=`getDeanGradList` &limit=`25`]]
          <p>Page [[+page]] of [[+pageCount]]</p>
          <div class="pagination">
          <ul id="pageNav">[[!+page.nav]]</ul>
          </div>
          [ed. note: dwillis210 last edited this post 13 years, 8 months ago.]
            • 22303 MODX Staff
            • 10,725 Posts
            I think you just want to issue a query and work with the result sets rather than returning objects here. You are not selecting unique rows representing objects, and in these cases you can prepare() and execute() the statements you build manually, e.g.
                $query = $modx->newQuery("Names");
                $query->distinct();
                $query->select(array($modx->getSelectColumns("Names", array("city"))));
                $query->where(array("category"=>$list_full, "termcode"=>$term));
                $query->sortby("city", "ASC");
            
                if ($query->prepare() && $query->stmt->execute()) {
                    foreach($query->stmt->fetchAll(PDO::FETCH_COLUMN) as $city)
                        $cityOptions .= $modx->getChunk("deangradlist-menu-city", array("city"=>$city->get("city")));
                }
              • 40735
              • 119 Posts
              I pasted your code into my snippet but I still get out of memory errors. If I change the getChunk() line to simply $cityOptions .= "<option>$city</option>"; it loads much quicker and populates the <select> tag like it should but instead of the city names that are in the database it populates it with a long list of mostly, but not entirely, four digit numbers. Maybe I shouldn't get 468 chunks in one snippet? I have no idea what the numbers are about though.

              Edit:
              Oh, actually, before I got the out of memory errors again it was telling me that $city had no get() method. It seems to just be a string. [ed. note: dwillis210 last edited this post 13 years, 8 months ago.]
                • 40735
                • 119 Posts
                Okay, we got it to work by modifying the code like so:
                $query = $modx->newQuery("Names");
                	$query->distinct();
                	//$query->select(array($modx->getSelectColumns("Names", array("city"))));
                    $query->select(array("city" => "city"));
                	$query->where(array("category"=>$list_full, "termcode"=>$term));
                	$query->sortby("city", "ASC");
                	 
                	if ($query->prepare() && $query->stmt->execute()){
                		foreach($query->stmt->fetchAll(PDO::FETCH_COLUMN) as $city)
                			$cityOptions .= "<option>$city</option>";
                	}


                It now populates the the <select> with the unique city names like we want but I'd like to understand what's going on here. What does getSelectColumns() do and why did it not work? Also, what is fetchAll(PDO::FETCH_COLUMN) doing? I would guess FETCH_COLUMN is a constant in the static PDO object but var_dump() reveals that it's an int with a value of 7. I have no idea what that's about.
                  • 22303 MODX Staff
                  • 10,725 Posts
                  Quote from: dwillis210 at Feb 14, 2013, 10:04 AM
                  What does getSelectColumns() do and why did it not work? Also, what is fetchAll(PDO::FETCH_COLUMN) doing? I would guess FETCH_COLUMN is a constant in the static PDO object but var_dump() reveals that it's an int with a value of 7. I have no idea what that's about.</select>
                  Sorry, I forgot to change the final part to reflect the change to using just the needed values from the result set.

                  Fetching objects requires that you at least have the primary key column selected in your query. If you are writing summary/reporting queries, like selecting distinct city names, you don't want a collection of objects representing rows in the database, you want a result set with the data—nothing more.

                  xPDOQuery->prepare() creates a PDOStatement object, which can then be used like any PDOStatement (learn more at http://php.net/manual/en/class.pdostatement.php). PDO::FETCH_COLUMN is a constant that controls how the fetch*() methods of a PDOStatement object fetch the data from the result set. In this case, FETCH_COLUMN means return an array representing all rows from the first column of the result set (or a specified column in the result set if you pass a second parameter indicating the column to fetch). All of the PDO constants are documented at http://www.php.net/manual/en/pdo.constants.php ...