Hi vickySFDC,

Here is a sample code for custom lookup :

// Image for lookup icon on click of which lookup window will open
<span class="lookupInput">
	<apex:inputText value="{!var.sObjName}" />
		<a href ="" style="cursor: pointer" onClick="javascript&colon;return openCompanyPopup(this.parentNode)">
			<img src="/s.gif" alt="Module Lookup (New Window)" class="lookupIcon" onblur="this.className = 'lookupIcon';" onfocus="this.className = 'lookupIconOn';" onmouseout="this.className = 'lookupIcon';this.className = 'lookupIcon';" onmouseover="this.className = 'lookupIconOn';this.className = 'lookupIconOn';" title="Module Lookup (New Window)" />
		</a>
	</apex:inputText>
</span>
   
//JS will execute after this click
function openCompanyPopup(obj)
{
	 currConColumn = obj;
	 var likeString = (obj.getElementsByTagName('input')[0]).value;
	 var url ;
	 url = "TBN_CustomLookUp?likestring=" + likeString +"&objName=Project_Module__c";
	 openWind(url);
	 return false;
}
 
//custom lookup vf page
<apex:page controller="TBN_CustomLookUpController" sidebar="false" showHeader="false">
<!--	<apex:SectionHeader Title="Conference Wizard" subTitle="Select Account" help="http://help.txt" />-->
      <script>      
         function populateParent(obj) 
         {
             var trNode = obj.parentNode.parentNode;
             var tdNodes = trNode.getElementsByTagName('td');
             var companyId = (tdNodes[0].getElementsByTagName('span')[0]).innerHTML;
             var companyName = (tdNodes[1].getElementsByTagName('a')[0]).innerHTML;
             companyName = companyName.replace('&amp;', '&');   
             top.window.opener.setSelectedCompanyDetails(companyId , companyName);      
             return false;
         }
     </script>
	<apex:form >
         <apex:outputPanel id="main">
             
             <apex:outputPanel rendered="true">  
                 <table id="tblSearch">
                     <tr>
                         <td><B>Search</B></td>
                         <td><apex:InputText value="{!likeString}"/></td>
                         <td><apex:CommandButton value="Go!" action="{!getCompany}" reRender="detail"/></td>
                     </tr>
                 </table>
             </apex:outputPanel>
             
             <br/>
             <br/>
                      
             <apex:outputPanel id="detail" >
                 <apex:pageBlock title="" rendered="{!showSearch}" >
                  <apex:outputPanel rendered ="{!if(showCompany.size > 0,true,false)}">
	                    <apex:pageBlockTable value="{!showCompany}" var="comp">
	                        <apex:column >
	                           <span style="display: none" width="0">{!comp['Id']}</span>
	                        </apex:column>
	                        <apex:column headerValue="Name">
	                            <apex:outputLink value="" onClick="return populateParent(this)">{!comp['Name']} </apex:outputLink> 
	                        </apex:column>
	                    </apex:pageBlockTable> 
                   </apex:outputPanel>
                   <apex:outputPanel rendered ="{!if(showCompany.size > 0,false,true)}">
                   	No Record
                   </apex:outputPanel>
                </apex:pageBlock>            
             </apex:outputPanel> 
             
        </apex:outputPanel>
         
    </apex:form> 
</apex:page>

//Controller for custom lookup
public with sharing class TBN_CustomLookUpController 
{	
  
    public String recordType {get;set;}
  	private String objName;
  	private String managerId;
  	private String strId;
  	private  string likeString1;
  	public String likeString {get;set;}
    
    public TBN_CustomLookUpController()
    {
    	likeString = '';
    	objName = System.currentPageReference().getParameters().get('objName');
    	managerId =  System.currentPageReference().getParameters().get('managerId');
    	strId = System.currentPageReference().getParameters().get('strId');
    	
	   	try
		{
			likeString = System.currentPageReference().getParameters().get('likestring');
		    getCompany();    
		}
		catch(exception ex)
		{
		}
    }
        
    public boolean showSearch
    {
        set
        {
           showSearch= value;
        }
        get
        {
          return showSearch;  
        }
    }
    
    public List<Sobject> getshowCompany()
    {
    	Set<String> setIdRB = new Set<String>();
    	if(strId != null)
    		setIdRB.addAll(strId.split(','));
    		
        List<Sobject> lstSobject = new List<Sobject>();
        
        String strQuery = '';
        if((likeString == '' || likeString == null) || likeString.trim().length() == 0)
        {
        	if(managerId != null)
        	{
        		strQuery = 'Select Id, Name From '+string.escapeSingleQuotes(objName)+' where Report_To__c = \'' + string.escapeSingleQuotes(managerId) + '\' and Id NOT IN: setIdRB ORDER BY Name';
        	}
        	else
            	strQuery = 'Select Id, Name From '+string.escapeSingleQuotes(objName)+' ORDER BY Name';
        }
        else
        {
            if(managerId != null)
            {
            	if(likeString1 != '% %')
            	{
            		strQuery = 'Select Id, Name From '+ string.escapeSingleQuotes(objName) +' where '+'Name like \'%' + string.escapeSingleQuotes(likeString) + '%\' and Report_To__c = \'' + string.escapeSingleQuotes(managerId) + '\''+ ' and Id NOT IN: setIdRB ORDER BY Name';
            	}
            	else
            		strQuery = 'Select Id, Name From '+string.escapeSingleQuotes(objName)+' where Report_To__c = \'' + string.escapeSingleQuotes(managerId) + '\' and Id NOT IN: setIdRB ORDER BY Name';
            }
            else
            {
            	strQuery = 'Select Id, Name From '+ string.escapeSingleQuotes(objName) +' where Name like \'%'+ string.escapeSingleQuotes(likeString) +'%\' ORDER BY Name';
            }
        }
        lstSobject =  Database.query(strQuery);
        
        if(managerId != null && !setIdRB.contains(managerId))
        {
        	list<User> lstUser = new list<User>();
        	if(likeString != '')
        		lstUser = [Select Id,Name from User where id = : string.escapeSingleQuotes(managerId) and Name like : likeString1 limit 1];
        	else
        		lstUser = [Select Id,Name from User where id = : string.escapeSingleQuotes(managerId) limit 1];
        		
        	if((lstUser.size() > 0 )&& (!setIdRB.contains(lstUser[0].Id)))
        	{
				lstSobject.add(lstUser[0]);
        	}
        }
        return lstSobject;
    }      
    public PageReference getCompany()
    {
        showSearch = true; 
        getshowCompany();
        return null;
    }
}

// add this JS in main page to receive the selected item from lookup
function setSelectedCompanyDetails(TTId, TTName)
{               
	var inputArr = currConColumn.getElementsByTagName('input');
	var test = currConColumn.parentNode.getElementsByTagName('input');
	inputArr[0].value =  TTName;
	inputArr[1].value = TTId;
	curPopupWindow.close();
	currConColumn = null;                      
}