I came across this post trying to find a solution to the same issue,
In my case i found that the popup was selecting the correct ID along the x axis but moving the position on the popup outside of the panel i wanted. So when i highlighted the linked item if it was say 800px on a scroller. It would move the popup to 800px outside the scroller.

This was the first issue I found. There were also ID issues along the X and Y for different commandLinks & when I reordered my page the popups would again be scrambled.

I worked out that the broken / buggy SalesForce code was within LookupHoverDetail.prototype.position. In here we were returning getObjX and getObjY wrongly.

So within my visualforce page I placed some overwriting code to replace the wrong possitions.

function getObjX(a) {
var left = 0; 
while(a){
left += (a.offsetLeft - a.scrollLeft + a.clientLeft);
a = a.offsetParent;
}
return left;
}
function getObjY(a) {
var top = 0; 
while(a){
top += (a.offsetTop - a.scrollTop + a.clientTop);
a = a.offsetParent;
}
return top;
}

Next I had to deal with the ID's being non-unique, I call this function after every Ajax event so that if I order the page, the Id's are to the correct links still.
This function will execute at the same time as the others on load.

function fixSalesforce() {
$FFDC('.lineItem A').each(function(index){
var me = $FFDC(this);
var aId = me.attr('id');
if (aId != null){
var trId = me.closest('tr').attr('id');
if (aId.indexOf(trId) != 0){
var newId = trId + aId;
me.attr('id', newId);
me.attr('onmouseover', me.attr('onmouseover').replace( new RegExp(aId, 'g' ), newId));
me.attr('onmouseout', me.attr('onmouseout').replace( new RegExp(aId, 'g' ), newId));
me.attr('onfocus', me.attr('onfocus').replace( new RegExp(aId, 'g' ), newId));
me.attr('onblur', me.attr('onblur').replace( new RegExp(aId, 'g' ), newId));
}
}
});
}

The aim of the above function is to make the ID's unique by prefixing them with the row ID which I put on the TR.

{
<apex:variable value="{!0}" var="tRow" /> 
<apex:repeat var="tl"value="{!viewState.BankStatement.TransactionLineItems}"id="tRow1"> 
<tr id="tRow{!tRow}" class="lineItem{!IF(tl.isChecked,'selected','')}">
<apex:variable var="tRow"value="{!tRow+1}"/>
}

Within all CommandLinks I then placed onComplete="Refresh();"

This is a function that will call the fixSalesforce() function from earlier.

I came across this post trying to find a solution to the same issue,
In my case i found that the popup was selecting the correct ID along the x axis but moving the position on the popup outside of the panel i wanted. So when i highlighted the linked item if it was say 800px on a scroller. It would move the popup to 800px outside the scroller.

This was the first issue I found. There were also ID issues along the X and Y for different commandLinks & when I reordered my page the popups would again be scrambled.

I worked out that the broken / buggy SalesForce code was within LookupHoverDetail.prototype.position. In here we were returning getObjX and getObjY wrongly.

So within my visualforce page I placed some overwriting code to replace the wrong possitions.

function getObjX(a) {
var left = 0; 
while(a){
left += (a.offsetLeft - a.scrollLeft + a.clientLeft);
a = a.offsetParent;
}
return left;
}
function getObjY(a) {
var top = 0; 
while(a){
top += (a.offsetTop - a.scrollTop + a.clientTop);
a = a.offsetParent;
}
return top;
}

Next I had to deal with the ID's being non-unique, I call this function after every Ajax event so that if I order the page, the Id's are to the correct links still.
This function will execute at the same time as the others on load.

function fixSalesforce() {
$FFDC('.lineItem A').each(function(index){
var me = $FFDC(this);
var aId = me.attr('id');
if (aId != null){
var trId = me.closest('tr').attr('id');
if (aId.indexOf(trId) != 0){
var newId = trId + aId;
me.attr('id', newId);
me.attr('onmouseover', me.attr('onmouseover').replace( new RegExp(aId, 'g' ), newId));
me.attr('onmouseout', me.attr('onmouseout').replace( new RegExp(aId, 'g' ), newId));
me.attr('onfocus', me.attr('onfocus').replace( new RegExp(aId, 'g' ), newId));
me.attr('onblur', me.attr('onblur').replace( new RegExp(aId, 'g' ), newId));
}
}
});
}

The aim of the above function is to make the ID's unique by prefixing them with the row ID which I put on the TR.

{
<apex:variable value="{!0}" var="tRow" /> 
<apex:repeat var="tl"value="{!viewState.BankStatement.TransactionLineItems}"id="tRow1"> 
<tr id="tRow{!tRow}" class="lineItem{!IF(tl.isChecked,'selected','')}">
<apex:variable var="tRow"value="{!tRow+1}"/>
}

Within all CommandLinks I then placed onComplete="Refresh();"

This is a function that will call the fixSalesforce() function from earlier.